lovecrafts · billhails · Oct 20, 2021 · Oct 26, 2021
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,14 @@
+.vscode
+*.code-workspace
+# temp files and debugging
+data
+archive
+algolia_product_import
+algolia_category_import.json
+ct_category_export.json
+ct_product_export.json
+product_importer_categories.json
+
 # Logs
 logs
 *.log

diff --git a/.sample.env b/.sample.env
@@ -1,29 +1,13 @@
 # commercetools
-CT_API_URL=https://api.commercetools.com
-CT_HOST=https://auth.sphere.io
-CT_AUTH_URL=https://auth.sphere.io
-CT_PROJECT_KEY=vsf-ct-dev
-CT_CLIENT_ID=RT4iJGDbDzZe4b2E6RyeNe9s
-CT_CLIENT_SECRECT=5eBt3yfZJWw1j7V6kXjfKXpuFP-YQXpg
-CT_ACCESS_TOKEN=RZehIyF9GGZgitL0LwHYK9SGPXLnAFAG
+CT_API_URL=https://api.europe-west1.gcp.commercetools.com/
+CT_HOST=https://auth.europe-west1.gcp.commercetools.com/
+CT_AUTH_URL=https://auth.europe-west1.gcp.commercetools.com/
+CT_PROJECT_KEY=lc-dev
+CT_CLIENT_ID=XXXXX
+CT_CLIENT_SECRECT=XXXXX
 
 #Algolia
-ALGOLIA_PRODUCTS_INDEX_NAME=products
-ALGOLIA_CATEGORY_INDEX_NAME=categories
-ALGOLIA_PROJECT_ID = RA46FT2Q5G
-ALGOLIA_WRITE_KEY = 69a1fff0143c7121bc2f35825680b272
-
-# Products per batch count should be < 500
-PRODUCTS_PER_BATCH=500
-
-#Attributes values taken by user
-VARIENT_ATTRIBUTES=color,commonSize,designer,size,style,gender
-PRODUCT_ATTRIBUTES=description
-
-LANGUAGES=en,de
-
-#Algolia timeout config in seconds
-ALGOLIA_CONNECT_TIMEOUT=180
-ALGOLIA_READ_TIMEOUT=180
-ALGOLIA_WRITE_TIMEOUT=180
-ALGOLIA_DNS_TIMEOUT=180
+ALGOLIA_PRODUCT_INDEX_NAME=lc_dev_ct_products
+ALGOLIA_CATEGORY_INDEX_NAME=lc_dev_ct_categories
+ALGOLIA_PROJECT_ID=XXXXX
+ALGOLIA_WRITE_KEY=XXXXX
diff --git a/.vscode/settings.json b/.vscode/settings.json
diff --git a/categories.json b/categories.json
diff --git a/category.js b/category.js
@@ -1,149 +1,127 @@
-import CategoryExporter from "@commercetools/category-exporter"
-import fs from 'fs'
-import dotenv from "dotenv";
-dotenv.config();
-import { sendtoalgolia } from './pushToAlgolia.js'
-var startBatching = new Date(),
-    endBatching, startConvertion, endConvertion, startSync, batchSync, endSync;
-const options = {
-    apiConfig: {
-        apiUrl: process.env.CT_API_URL,
-        host: process.env.CT_HOST,
-        authUrl: process.env.CT_AUTH_URL,
-        projectKey: process.env.CT_PROJECT_KEY,
-        credentials: {
-            clientId: process.env.CT_CLIENT_ID,
-            clientSecret: process.env.CT_CLIENT_SECRECT,
-        }
-    },
-
-}
-
-const logger = {
-    error: console.error,
-    warn: console.warn,
-    info: console.log,
-    debug: console.debug,
+import CategoryExporter from "@commercetools/category-exporter";
+import fs from "fs";
+import { pushToAlgolia } from "./pushToAlgolia.js";
+import { logger, getTranslation } from "./utils.js";
+import config from "./config.js";
+import util from 'util';
+
+const categoryExporter = new CategoryExporter.default({ apiConfig: config.commerceToolsAuth }, logger);
+
+const outputStream = fs.createWriteStream(config.ctCategoryDumpFileName);
+
+const makeCategoryMap = categories => categories.reduce((acc, cat) => ({...acc, [cat.id]: cat }), {});
+
+const writeFile = (fileName, data) => {
+  fs.writeFile(
+    fileName,
+    JSON.stringify(data),
+    "utf8",
+    function(err) {
+      if (err) {
+        logger.error(err);
+      }
+    }
+  );
 }
 
-const categoryExporter = new CategoryExporter.default(options, logger)
-var outputStream = fs.createWriteStream('categories.txt');
-// Register error listener
-outputStream.on('error', function(v) {
-    console.log(v);
-})
-
-outputStream.on('finish', () => {
-    endBatching = new Date() - startBatching;
-    startConvertion = Date();
-    fs.readFile('categories.txt', 'utf8', (err, data) => {
-        if (err) {
-            console.error(err)
-            return
+const makeCategories = (ctCategories, categoryMap) => {
+  const productImporterCategories = [];
+  const algoliaCategories = [];
+
+  for (const ctCategory of ctCategories) {
+    const algoliaCategory = {
+      id: ctCategory.id,
+      version: ctCategory.version,
+      name: ctCategory.name,
+      objectID: ctCategory.id,
+      path: {},
+      slug: {},
+    };
+    const productImporterCategory = {
+      id: ctCategory.id,
+      categories: {},
+      hierarchicalCategories: {},
+    };
+
+    const current = { typeId: "category", id: ctCategory.id }
+
+    for (const locale of config.locales) {
+      productImporterCategory.categories[locale] = {};
+      productImporterCategory.hierarchicalCategories[locale] = {};
+      algoliaCategory.name[locale] = getTranslation(locale, algoliaCategory.name);
+      const slugArray = [];
+      const pathArray = [];
+
+      for (const ancestor of [...ctCategory.ancestors, current]) {
+        const ancestorCategory = categoryMap[ancestor.id];
+        const name = getTranslation(locale, ancestorCategory.name);
+        const slug = getTranslation(locale, ancestorCategory.slug);
+
+        if (name) {
+          pathArray.push(name);
+        } else {
+          logger.error(`failed to find name for locale ${locale}`, ancestorCategory);
+          process.exit(1);
         }
-        var categories = JSON.parse(data);
-        var finalcategories = [];
-        for (let obj of categories) {
-            var category = {};
-            category.id = obj.id;
-            category.version = obj.version;
-            category.name = obj.name;
-            category.objectID = obj.id;
-            category.path = {};
-            category.slug = {};
-            let env_languages = (process.env.LANGUAGES).split(',');
-            let i = 0;
-            obj.ancestors.push({ typeId: 'category', id: obj.id });
-            if (obj.ancestors.length != 0) {
-                for (let lan of env_languages) {
-                    category.path[lan] = '';
-                    category.slug[lan] = '';
-                }
-            }
-
-            for (let lan of env_languages) {
-                let path_str, slug_str = "";
-                for (let _cat_obj of obj.ancestors) {
-                    var cat = categories.filter(x => x.id == _cat_obj.id)[0];
-
-                    if (slug_str == "") {
-                        slug_str = cat.slug[lan];
-                        path_str = cat.name[lan];
-                    } else {
-                        slug_str += '/' + cat.slug[lan];
-                        path_str += ' > ' + cat.name[lan];
-                    }
-                }
-                category.slug[lan] = slug_str;
-                category.path[lan] = path_str;
-            }
-            finalcategories.push(category)
 
+        if (slug) {
+          slugArray.push(slug);
+        } else {
+          logger.error(`failed to find slug for locale ${locale}`, ancestorCategory);
+          process.exit(1);
         }
 
-        var env_languages = (process.env.LANGUAGES).split(',');
-        var resultcategories = [];
-        for (let category of categories) {
-            var _resultObject = {};
-            _resultObject.id = category.id;
-            _resultObject.categories = {};
-            _resultObject.hierarchicalCategories = {};
-            for (let lan of env_languages) {
-                _resultObject.categories[lan] = {};
-                _resultObject.hierarchicalCategories[lan] = {};
-            }
-            for (let lan of env_languages) {
-                let slug_str = "";
-                let path_str = "";
-                let path_str_hie = "";
-                let level = 0;
-                for (let ancestor of category.ancestors) {
-                    let _cat_obj = categories.filter(x => x.id === ancestor.id)[0];
-
-                    if (slug_str == "") {
-                        slug_str = _cat_obj.slug[lan];
-                        path_str = _cat_obj.name[lan];
-                        path_str_hie = _cat_obj.name[lan];
-
-                    } else {
-                        slug_str += '/' + _cat_obj.slug[lan];
-                        path_str += ' > ' + _cat_obj.name[lan];
-                        path_str_hie += ' > ' + _cat_obj.name[lan];
-                    }
-                    _resultObject.hierarchicalCategories[lan]['lvl' + level] = path_str_hie;
-
-                    level++;
-                }
-                _resultObject.hierarchicalCategories[lan]
-
-                _resultObject.categories[lan]['slug'] = slug_str;
-                _resultObject.categories[lan]['path'] = path_str;
-
-            }
-            resultcategories.push(_resultObject)
-        }
+        productImporterCategory.hierarchicalCategories[locale]['lvl' + (pathArray.length - 1)] = pathArray.join(' > ');
+      }
 
-        fs.writeFile('categories.json', JSON.stringify(resultcategories), 'utf8', function(err) {
-            if (err) {
-                return console.log(err);
-            }
-
-        });
-        if (finalcategories) {
-            endConvertion = new Date() - startConvertion;
-            sendtoalgolia(startBatching, endBatching, endSync, 'category', 'categories.json', finalcategories, process.env.ALGOLIA_CATEGORY_INDEX_NAME)
-                // fs.writeFile('finalcategories.json', JSON.stringify(finalcategories), 'utf8', function(err) {
-                //     if (err) {
-                //         return console.log(err);
-                //     }
-
-            // });
-        }
-    })
-})
+      if (pathArray.length) {
+        productImporterCategory.categories[locale].path = pathArray.join(' > ');
+        algoliaCategory.path[locale] = pathArray.join(' > ');
+      }
+
+      if (slugArray.length) {
+        productImporterCategory.categories[locale].slug = slugArray.join('/');
+        algoliaCategory.slug[locale] = slugArray.join('/');
+      }
+    }
+
+    productImporterCategories.push(productImporterCategory);
+    algoliaCategories.push(algoliaCategory);
+  }
 
-console.log("Category Indexer Executing ..." + '\n')
-console.time('Category Indexer code execution time:');
+  return { productImporterCategories, algoliaCategories };
+}
+
+// Register error listener
+outputStream.on("error", v => logger.error(v));
+
+outputStream.on("finish", () => {
+  fs.readFile(config.ctCategoryDumpFileName, "utf8", (err, data) => {
+    if (err) {
+      logger.error(err);
+      return;
+    }
+
+    const ctCategories = JSON.parse(data);
+    const categoryMap = makeCategoryMap(ctCategories);
+    const { algoliaCategories, productImporterCategories } = makeCategories(ctCategories, categoryMap);
+
+    writeFile(config.productCategoriesDumpFileName, productImporterCategories);
+    writeFile(config.categoryDebugDumpFileName, algoliaCategories);
+
+    if (false && algoliaCategories) {
+      pushToAlgolia({
+        syncfrom: "category",
+        filename: config.categoryDebugDumpFileName,
+        data: algoliaCategories,
+        indexname: config.algoliaCategoryIndexName,
+        logger
+      });
+    }
+  });
+});
+
+logger.info("Category Indexer Executing");
+console.time("Category Indexer code execution time:");
 categoryExporter.run(outputStream);
-console.timeEnd('Category Indexer code execution time:');
-console.log("-----------------------------------------------");
+console.timeEnd("Category Indexer code execution time:");
diff --git a/config.js b/config.js
@@ -0,0 +1,60 @@
+import dotenv from "dotenv";
+dotenv.config();
+
+const config = {
+  ctCategoryDumpFileName: "ct_category_export.json",
+  ctProductDumpFileName: "ct_product_export.json",
+  productCategoriesDumpFileName: "product_importer_categories.json",
+  categoryDebugDumpFileName: "algolia_category_import.json",
+  locales: ['en-GB', 'en-US', 'en-AU', 'de-DE'],
+  localeFallbacks: { // order to search translations if target is missing (non-recursive)
+    'en-GB': ['en-US', 'en-AU'],
+    'en-US': ['en-GB', 'en-AU'],
+    'en-AU': ['en-GB', 'en-US'],
+    'de-DE': ['en-GB', 'en-US', 'en-AU']
+  },
+  algoliaProjectId: process.env.ALGOLIA_PROJECT_ID,
+  algoliaWriteKey: process.env.ALGOLIA_WRITE_KEY,
+  algoliaCategoryIndexName: process.env.ALGOLIA_CATEGORY_INDEX_NAME,
+  algoliaProductIndexName: process.env.ALGOLIA_PRODUCT_INDEX_NAME,
+  algoliaConnectTimeout: 180,
+  algoliaReadTimeout: 180,
+  algoliaWriteTimeout: 180,
+  algoliaDnsTimeout: 180,
+  ctApiUrl: process.env.CT_API_URL,
+  ctAuthUrl: process.env.CT_AUTH_URL,
+  ctHost: process.env.CT_HOST,
+  ctProjectKey: process.env.CT_PROJECT_KEY,
+  ctClientId: process.env.CT_CLIENT_ID,
+  ctClientSecret: process.env.CT_CLIENT_SECRECT,
+  productsPerBatch: 500,
+  productAttributes: ["shippingRestrictions"],
+  varientAttributes: [
+    "brand",
+    "colourButtons",
+    "craft",
+    "crochetHookSize",
+    "isDesigner",
+    "isVirtual",
+    "lengthNeedles",
+    "needleSize",
+    "primaryCraft",
+    "subtitle",
+    "taxjarCategoryCode",
+    "type",
+  ],
+};
+
+export default {
+  ...config,
+  commerceToolsAuth: {
+    apiUrl: config.ctApiUrl,
+    host: config.ctHost,
+    authUrl: config.ctAuthUrl,
+    projectKey: config.ctProjectKey,
+    credentials: {
+      clientId: config.ctClientId,
+      clientSecret: config.ctClientSecret
+    }
+  }
+};