Skip to content

EDU-13262: Mapping errors from Contentful scripts #127

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 37 additions & 5 deletions docs-utils/fix-callouts.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,30 @@
const fs = require('fs').promises;
const fsSync = require('fs');
const path = require('path');

const scriptErrorsPath = path.resolve(__dirname, '../script-errors.md');

// Function to log errors to script-errors.md
async function logScriptStatus(scriptName, status, errors = []) {
const timestamp = new Date().toISOString();
const errorDetails = errors.length
? errors.map(err => `| ${scriptName} | ❌ Failed | ${err} | ${timestamp} |`).join('\n')
: `| ${scriptName} | ✅ Success | - | ${timestamp} |`;

const header = `| Script Name | Status | Details | Timestamp |\n|-------------|--------|---------|-----------|`;
const content = `${header}\n${errorDetails}\n`;

try {
if (!fsSync.existsSync(scriptErrorsPath)) {
fsSync.writeFileSync(scriptErrorsPath, content);
} else {
fsSync.appendFileSync(scriptErrorsPath, `${errorDetails}\n`);
}
} catch (err) {
console.error(`Error logging script status: ${err.message}`);
}
}

// The root folder to process
const rootFolder = 'docs';

Expand Down Expand Up @@ -112,12 +136,20 @@ async function processDirectory(directory) {
}

async function fixCallouts() {
console.log("Fixing callouts...");
await processDirectory(rootFolder);
while (activeFiles > 0 || fileQueue.length > 0) {
await new Promise(resolve => setTimeout(resolve, 100));
const errors = [];
try {
console.log("Fixing callouts...");
await processDirectory(rootFolder);
while (activeFiles > 0 || fileQueue.length > 0) {
await new Promise(resolve => setTimeout(resolve, 100));
}
console.log("Finished fixing callouts in markdown files.");
} catch (err) {
errors.push(err.message);
console.error(`Error fixing callouts: ${err.message}`);
} finally {
await logScriptStatus('fixCallouts', errors.length ? 'Failed' : 'Success', errors);
}
console.log("Finished fixing callouts in markdown files.");
}

module.exports = { fixCallouts };
45 changes: 38 additions & 7 deletions docs-utils/replace-quotes.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,35 @@
const fs = require('fs').promises;
const fsSync = require('fs');
const path = require('path');

const docsDir = path.resolve(__dirname, '../docs');
const scriptErrorsPath = path.resolve(__dirname, '../script-errors.md');
const MAX_CONCURRENT_FILES = 100;

let activeFiles = 0;
let fileQueue = [];

// Function to log errors to script-errors.md
async function logScriptStatus(scriptName, status, errors = []) {
const timestamp = new Date().toISOString();
const errorDetails = errors.length
? errors.map(err => `| ${scriptName} | ❌ Failed | ${err} | ${timestamp} |`).join('\n')
: `| ${scriptName} | ✅ Success | - | ${timestamp} |`;

const header = `| Script Name | Status | Details | Timestamp |\n|-------------|--------|---------|-----------|`;
const content = `${header}\n${errorDetails}\n`;

try {
if (!fsSync.existsSync(scriptErrorsPath)) {
fsSync.writeFileSync(scriptErrorsPath, content);
} else {
fsSync.appendFileSync(scriptErrorsPath, `${errorDetails}\n`);
}
} catch (err) {
console.error(`Error logging script status: ${err.message}`);
}
}

// Function to process each Markdown file
async function processFile(filePath) {
try {
Expand Down Expand Up @@ -84,15 +107,23 @@ async function processDirectory(dirPath) {
}

async function replaceQuotes() {
console.log("Replacing quotation marks...");
await processDirectory(docsDir);
const errors = [];
try {
console.log("Replacing quotation marks...");
await processDirectory(docsDir);

// Wait until all files are processed
while (activeFiles > 0 || fileQueue.length > 0) {
await new Promise(resolve => setTimeout(resolve, 100));
}
// Wait until all files are processed
while (activeFiles > 0 || fileQueue.length > 0) {
await new Promise(resolve => setTimeout(resolve, 100));
}

console.log("Finished replacing quotation marks in markdown files.");
console.log("Finished replacing quotation marks in markdown files.");
} catch (err) {
errors.push(err.message);
console.error(`Error replacing quotes: ${err.message}`);
} finally {
await logScriptStatus('replaceQuotes', errors.length ? 'Failed' : 'Success', errors);
}
}

module.exports = { replaceQuotes };
51 changes: 41 additions & 10 deletions docs-utils/update-all-images.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
const fs = require('fs');
const fsSync = require('fs');
const path = require('path');

const docsDirEN = path.join(__dirname, '../docs/en/');
const docsDirPT = path.join(__dirname, '../docs/pt/');
const docsDirES = path.join(__dirname, '../docs/es/');
const scriptErrorsPath = path.resolve(__dirname, '../script-errors.md');

const { updateImages } = require('./update-images.js');

Expand All @@ -12,6 +14,27 @@ const MAX_CONCURRENT_FILES = 100;
let activeFiles = 0;
let fileQueue = [];

// Function to log errors to script-errors.md
async function logScriptStatus(scriptName, status, errors = []) {
const timestamp = new Date().toISOString();
const errorDetails = errors.length
? errors.map(err => `| ${scriptName} | ❌ Failed | ${err} | ${timestamp} |`).join('\n')
: `| ${scriptName} | ✅ Success | - | ${timestamp} |`;

const header = `| Script Name | Status | Details | Timestamp |\n|-------------|--------|---------|-----------|`;
const content = `${header}\n${errorDetails}\n`;

try {
if (!fsSync.existsSync(scriptErrorsPath)) {
fsSync.writeFileSync(scriptErrorsPath, content);
} else {
fsSync.appendFileSync(scriptErrorsPath, `${errorDetails}\n`);
}
} catch (err) {
console.error(`Error logging script status: ${err.message}`);
}
}

// Function to process each Markdown file
async function processFile(filePath) {
try {
Expand Down Expand Up @@ -56,17 +79,25 @@ async function processDirectory(dirPath) {

// Start iterating from the target directory and process articles sequentially
async function updateAllImages() {
console.log("Updating all images...");
await processDirectory(docsDirEN);
await processDirectory(docsDirPT);
await processDirectory(docsDirES);
const errors = [];
try {
console.log("Updating all images...");
await processDirectory(docsDirEN);
await processDirectory(docsDirPT);
await processDirectory(docsDirES);

// Wait until all files are processed
while (activeFiles > 0 || fileQueue.length > 0) {
await new Promise(resolve => setTimeout(resolve, 100));
}
// Wait until all files are processed
while (activeFiles > 0 || fileQueue.length > 0) {
await new Promise(resolve => setTimeout(resolve, 100));
}

console.log("Finished replacing all images in markdown files.");
console.log("Finished replacing all images in markdown files.");
} catch (err) {
errors.push(err.message);
console.error(`Error updating images: ${err.message}`);
} finally {
await logScriptStatus('updateAllImages', errors.length ? 'Failed' : 'Success', errors);
}
}

(async () => {
Expand All @@ -77,4 +108,4 @@ async function updateAllImages() {
}
})();

module.exports = { updateAllImages }
module.exports = { updateAllImages };
136 changes: 63 additions & 73 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const fs = require('fs').promises;
const fsSync = require('fs');
const path = require('path');
const docsFolderPath = path.join(__dirname, 'docs');
const scriptErrorsPath = path.join(__dirname, 'script-errors.md');

const contentful = require("contentful-management");
require("dotenv").config();
Expand Down Expand Up @@ -38,26 +39,48 @@ let contentTypes = {
unknownContentTypes: [],
};

// Function to log errors to script-errors.md
async function logScriptStatus(scriptName, status, errors = []) {
const timestamp = new Date().toISOString();
const errorDetails = errors.length
? errors.map(err => `| ${scriptName} | ❌ Failed | ${err} | ${timestamp} |`).join('\n')
: `| ${scriptName} | ✅ Success | - | ${timestamp} |`;

const header = `| Script Name | Status | Details | Timestamp |\n|-------------|--------|---------|-----------|`;
const content = `${header}\n${errorDetails}\n`;

try {
if (!fsSync.existsSync(scriptErrorsPath)) {
fsSync.writeFileSync(scriptErrorsPath, content);
} else {
fsSync.appendFileSync(scriptErrorsPath, `${errorDetails}\n`);
}
} catch (err) {
console.error(`Error logging script status: ${err.message}`);
}
}

// Function to delete all markdown files in the 'docs' folder and its subfolders
async function deleteMarkdownFiles(folderPath, isTopLevel = true) {
const errors = [];
if (isTopLevel) {
console.log("Deleting files and folders...");
}

async function processFiles(files) {
const fileDeletions = files.map(async file => {
const filePath = path.join(folderPath, file);
const stats = await fs.stat(filePath);

if (stats.isDirectory()) {
// Recursively delete markdown files in subdirectories
await deleteMarkdownFiles(filePath, false);
await fs.rmdir(filePath); // Delete folder after processing
// console.log(`Deleted folder: ${filePath}`);
} else {
// Delete markdown files
await fs.unlink(filePath);
// console.log(`Deleted: ${filePath}`);
try {
const stats = await fs.stat(filePath);

if (stats.isDirectory()) {
await deleteMarkdownFiles(filePath, false);
await fs.rmdir(filePath);
} else {
await fs.unlink(filePath);
}
} catch (err) {
errors.push(`Error deleting file/folder: ${filePath} - ${err.message}`);
}
});

Expand All @@ -71,7 +94,10 @@ async function deleteMarkdownFiles(folderPath, isTopLevel = true) {
console.log("All files and folders in the docs folder have been deleted.");
}
} catch (err) {
errors.push(`Error reading folder: ${folderPath} - ${err.message}`);
console.error(`Error during deletion: ${err.message}`);
} finally {
await logScriptStatus('deleteMarkdownFiles', errors.length ? 'Failed' : 'Success', errors);
}
}

Expand Down Expand Up @@ -163,77 +189,31 @@ async function getSubcategories() {
return subcategories;
}

// Wrap getEntries with error logging
async function getEntries() {

await getCategories();
await getSubcategories();

console.log("Getting entries from Contentful...");
const errors = [];
try {
await getCategories();
await getSubcategories();

console.log("Getting entries from Contentful...");
const space = await client.getSpace("alneenqid6w5");
const environment = await space.getEnvironment("master");

let skip = 0;
let limit = 100;
let totalCount = 0;

let skip = 0, limit = 100, totalCount = 0;
do {
const response = await environment.getEntries({ skip, limit });
const entries = response;
totalCount = response.total;

for (let j = 0; j < entries.items.length ; j++) {
let entry = entries.items[j];
// console.log(entry.fields)
createMarkdownFile(entry, categories, subcategories);
}
response.items.forEach(entry => createMarkdownFile(entry, categories, subcategories));
skip += limit;
} while (skip < totalCount);
console.log(
"Total amount of entries retrieved from Contentful:",
totalCount
);
console.log("Authors:", contentTypes.authors.length);
console.log("Tracks:", contentTypes.tracks.length);
console.log("Known Issues:", contentTypes.knownIssues.length);
console.log("Tutorials:", contentTypes.tutorials.length);
console.log("Track Articles:", contentTypes.trackArticles.length);
console.log("Track Topics:", contentTypes.trackTopics.length);
console.log("FAQs:", contentTypes.faqs.length);
console.log("Announcements:", contentTypes.announcements.length);
console.log("Categories:", contentTypes.categories.length);
console.log("Subcategories:", contentTypes.subcategories.length);
console.log("Admin V4 docs (deprecated):", contentTypes.adminV4docs.length);
console.log("TopBar (deprecated):", contentTypes.topBar.length);
console.log("TopQuestions (deprecated):", contentTypes.topQuestions.length);
console.log("Concepts (deprecated):", contentTypes.concepts.length);
console.log(
"Business Guides (deprecated):",
contentTypes.businessGuides.length
);
console.log(
"Unknown Content Type:",
contentTypes.unknownContentTypes.length
);
console.log("Entries that generated files:", fileCount / 3); // fileCount divided by the amount of locales, since it creates one file per locale
console.log(
"Amount of errors (difference between total of entries, entries that generated files and correctly ignored entries):",
totalCount -
fileCount / 3 -
contentTypes.authors.length -
contentTypes.tracks.length -
contentTypes.trackTopics.length -
contentTypes.categories.length -
contentTypes.subcategories.length -
contentTypes.adminV4docs.length -
contentTypes.topBar.length -
contentTypes.topQuestions.length -
contentTypes.concepts.length -
contentTypes.businessGuides.length -
contentTypes.unknownContentTypes.length
);
} catch (error) {
console.log("Error occurred while fetching entry:", error);

console.log("Entries fetched and files generated successfully.");
} catch (err) {
errors.push(`Error fetching entries: ${err.message}`);
console.error(`Error fetching entries: ${err.message}`);
} finally {
await logScriptStatus('getEntries', errors.length ? 'Failed' : 'Success', errors);
}
}

Expand Down Expand Up @@ -834,4 +814,14 @@ async function main() {
}
}

main();
// Remove the automatic execution of the main function
// main();

// Export individual functions for independent execution
module.exports = {
deleteMarkdownFiles,
getEntries,
replaceQuotes,
fixCallouts,
updateAllImages
};