refactor(client): speed up client updates (#65025)

client/gatsby-node.js

@@ -8,7 +8,6 @@ const webpack = require('webpack');
 const { SuperBlocks } = require('@freecodecamp/shared/config/curriculum');
 const env = require('./config/env.json');
 const {
-  createChallengePages,
   createBlockIntroPages,
   createSuperBlockIntroPages
 } = require('./utils/gatsby');
@@ -60,62 +59,11 @@ exports.createPages = async function createPages({
 
   const result = await graphql(`
     {
-      allChallengeNode(
+      allChallengeNode {
-        sort: {
-          fields: [
-            challenge___superOrder
-            challenge___order
-            challenge___challengeOrder
-          ]
-        }
-      ) {
         edges {
           node {
-            id
             challenge {
               block
-              blockLabel
-              blockLayout
-              certification
-              challengeType
-              dashedName
-              demoType
-              disableLoopProtectTests
-              disableLoopProtectPreview
-              fields {
-                slug
-                blockHashSlug
-              }
-              id
-              isLastChallengeInBlock
-              order
-              required {
-                link
-                src
-              }
-              challengeOrder
-              challengeFiles {
-                name
-                ext
-                contents
-                head
-                tail
-                history
-                fileKey
-              }
-              saveSubmissionToDB
-              solutions {
-                contents
-                ext
-                history
-                fileKey
-              }
-              superBlock
-              superOrder
-              template
-              usesMultifileEditor
-              chapter
-              module
             }
           }
         }
@@ -140,40 +88,6 @@ exports.createPages = async function createPages({
     }
   `);
 
-  const allChallengeNodes = result.data.allChallengeNode.edges.map(
-    ({ node }) => node
-  );
-
-  const createIdToNextPathMap = nodes =>
-    nodes.reduce((map, node, index) => {
-      const nextNode = nodes[index + 1];
-      const nextPath = nextNode ? nextNode.challenge.fields.slug : null;
-      if (nextPath) map[node.id] = nextPath;
-      return map;
-    }, {});
-
-  const createIdToPrevPathMap = nodes =>
-    nodes.reduce((map, node, index) => {
-      const prevNode = nodes[index - 1];
-      const prevPath = prevNode ? prevNode.challenge.fields.slug : null;
-      if (prevPath) map[node.id] = prevPath;
-      return map;
-    }, {});
-
-  const idToNextPathCurrentCurriculum =
-    createIdToNextPathMap(allChallengeNodes);
-
-  const idToPrevPathCurrentCurriculum =
-    createIdToPrevPathMap(allChallengeNodes);
-
-  // Create challenge pages.
-  result.data.allChallengeNode.edges.forEach(
-    createChallengePages(createPage, {
-      idToNextPathCurrentCurriculum,
-      idToPrevPathCurrentCurriculum
-    })
-  );
-
   const blocks = uniq(
     result.data.allChallengeNode.edges.map(
       ({

client/utils/build-challenges.js

@@ -51,7 +51,6 @@ exports.replaceChallengeNodes = () => {
     const block = path.basename(parentDir);
     const filename = path.basename(filePath);
 
-    console.log(`Replacing challenge nodes for ${filePath}`);
     const meta = getBlockStructure(block);
     const superblocks = getSuperblocks(block);
 

client/utils/gatsby/challenge-page-creator.js

@@ -69,23 +69,25 @@ const views = {
   examDownload
 };
 
-function getIsFirstStepInBlock(id, edges) {
+function getIsFirstStepInBlock(id, nodes) {
-  const current = edges[id];
+  const current = nodes[id];
-  const previous = edges[id - 1];
+  const previous = nodes[id - 1];
 
   if (!previous) return true;
-  return previous.node.challenge.block !== current.node.challenge.block;
+  return previous.challenge.block !== current.challenge.block;
 }
 
 function getTemplateComponent(challengeType) {
   return views[viewTypes[challengeType]];
 }
 
+exports.getTemplateComponent = getTemplateComponent;
+
 exports.createChallengePages = function (
   createPage,
   { idToNextPathCurrentCurriculum, idToPrevPathCurrentCurriculum }
 ) {
-  return function ({ node }, index, allChallengeEdges) {
+  return function (node, index, allChallengeNodes) {
     const {
       dashedName,
       disableLoopProtectTests,
@@ -118,7 +120,7 @@ exports.createChallengePages = function (
           chapter,
           module,
           block,
-          isFirstStep: getIsFirstStepInBlock(index, allChallengeEdges),
+          isFirstStep: getIsFirstStepInBlock(index, allChallengeNodes),
           template,
           required,
           isLastChallengeInBlock: isLastChallengeInBlock,
@@ -129,7 +131,7 @@ exports.createChallengePages = function (
         },
         projectPreview: getProjectPreviewConfig(
           node.challenge,
-          allChallengeEdges
+          allChallengeNodes
         ),
         id: node.id
       }
@@ -140,12 +142,12 @@ exports.createChallengePages = function (
 // TODO: figure out a cleaner way to get the last challenge in a block. Create
 // it during the curriculum build process and attach it to the first challenge?
 // That would remove the need to analyse allChallengeEdges.
-function getProjectPreviewConfig(challenge, allChallengeEdges) {
+function getProjectPreviewConfig(challenge, allChallengeNodes) {
   const { block } = challenge;
 
-  const challengesInBlock = allChallengeEdges
+  const challengesInBlock = allChallengeNodes
-    .filter(({ node: { challenge } }) => challenge.block === block)
+    .filter(({ challenge }) => challenge.block === block)
-    .map(({ node: { challenge } }) => challenge);
+    .map(({ challenge }) => challenge);
   const lastChallenge = challengesInBlock[challengesInBlock.length - 1];
   const solutionFiles = lastChallenge.solutions[0] ?? [];
   const lastChallengeFiles = lastChallenge.challengeFiles ?? [];

curriculum/schema/challenge-schema.js

@@ -342,6 +342,8 @@ const schema = Joi.object().keys({
       })
   }),
   showSpeakingButton: Joi.bool(),
+  // This is only to be used for dynamic client updates.
+  sourceLocation: Joi.string(),
   solutions: Joi.array().items(Joi.array().items(fileJoi).min(1)),
   superBlock: Joi.string().regex(slugWithSlashRE),
   superOrder: Joi.number(),

curriculum/src/build-superblock.ts

@@ -1,5 +1,5 @@
 import { existsSync, readdirSync } from 'fs';
-import { resolve } from 'path';
+import { join, resolve, basename } from 'path';
 import { isEmpty } from 'lodash';
 import debug from 'debug';
 
@@ -336,6 +336,13 @@ export class BlockCreator {
     );
 
     challenge.translationPending = this.lang !== 'english' && !isAudited;
+    // Add source location to allow tracing back to original file (necessary to
+    // update the client when files change)
+    challenge.sourceLocation = join(
+      basename(this.blockContentDir),
+      block,
+      filename
+    );
 
     return finalizeChallenge(challenge, meta);
   }

tools/client-plugins/gatsby-source-challenges/gatsby-node.js

@@ -1,4 +1,5 @@
 const chokidar = require('chokidar');
+
 const {
   getSuperblockStructure
 } = require('@freecodecamp/curriculum/file-handler');
@@ -7,6 +8,23 @@ const {
 } = require('@freecodecamp/curriculum/build-curriculum');
 
 const { createChallengeNode } = require('./create-challenge-nodes');
+const {
+  createChallengePages,
+  getTemplateComponent
+} = require('../../../client/utils/gatsby');
+
+// createPagesStatefully only runs once, but we need the following when
+// updating challenges, so they have to be stored in memory.
+let allChallengeNodes;
+let idToNextPathCurrentCurriculum;
+let idToPrevPathCurrentCurriculum;
+const filepathToStatefullyCreatedNodes = new Map();
+const filePathToCreatedNodes = new Map();
+// reverse lookup, to detect if an updated file has "overwritten" another file
+// (i.e. the updated file now has the same node id as another file).
+const idToFilepath = new Map();
+// recently overwritten files
+const idToOverwrittenFile = new Map();
 
 exports.sourceNodes = function sourceChallengesSourceNodes(
   { actions, reporter, createNodeId, createContentDigest },
@@ -31,7 +49,7 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
     a path to a curriculum directory
     `);
   }
-  const { createNode } = actions;
+  const { createNode, deleteNode, deletePage } = actions;
   const watcher = chokidar.watch(curriculumPath, {
     ignored: /(^|[/\\])\../,
     ignoreInitial: true,
@@ -39,16 +57,93 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
     cwd: curriculumPath
   });
 
+  function deletePages(filePath) {
+    const statefulNodes = filepathToStatefullyCreatedNodes.get(filePath) || [];
+    statefulNodes.forEach(node => {
+      deleteNode(node);
+      deletePage({
+        path: node.challenge.fields.slug,
+        component: getTemplateComponent(node.challenge.challengeType)
+      });
+      idToFilepath.delete(node.id);
+    });
+
+    const createdNodes = filePathToCreatedNodes.get(filePath) || [];
+    createdNodes.forEach(node => {
+      deleteNode(node);
+      idToFilepath.delete(node.id);
+    });
+
+    filepathToStatefullyCreatedNodes.delete(filePath);
+    filePathToCreatedNodes.delete(filePath);
+  }
+
+  function tryToDeletePages(filePath) {
+    const oldCreatedNodeIds = (filePathToCreatedNodes.get(filePath) ?? []).map(
+      node => node.id
+    );
+
+    const oldStatefullyCreatedNodeIds = (
+      filepathToStatefullyCreatedNodes.get(filePath) ?? []
+    ).map(node => node.id);
+
+    const oldNodeIds = [...oldCreatedNodeIds, ...oldStatefullyCreatedNodeIds];
+
+    const overwrittenFiles = new Set(
+      oldNodeIds.map(id => idToOverwrittenFile.get(id))
+    );
+
+    if (overwrittenFiles.has(filePath)) {
+      // since this has already been overwritten, it doesn't need
+      // deleting, but there's no longer any need to track that it was
+      // overwritten.
+      oldNodeIds.forEach(id => {
+        idToOverwrittenFile.delete(id);
+      });
+    } else {
+      deletePages(filePath);
+    }
+  }
+
   function handleChallengeUpdate(filePath, action = 'changed') {
+    if (action === 'deleted') {
+      // We have to return before calling onSourceChange, since the file is
+      // gone.
+      return tryToDeletePages(filePath);
+    }
+
     return onSourceChange(filePath)
       .then(challenges => {
         const actionText = action === 'added' ? 'creating' : 'replacing';
         reporter.info(
           `Challenge file ${action}: ${filePath}, ${actionText} challengeNodes with ids ${challenges.map(({ id }) => id).join(', ')}`
         );
-        challenges.forEach(challenge =>
+
-          createVisibleChallenge(challenge, { isReloading: true })
+        if (action === 'changed') {
+          tryToDeletePages(filePath);
+        }
+
+        const challengeNodes = challenges.map(challenge =>
+          reportNodeCreationToGatsby(challenge, {
+            isReloading: true
+          })
         );
+
+        // Track if file has been overwritten.
+        challengeNodes.forEach(({ id }) => {
+          const maybeFilepath = idToFilepath.get(id);
+          if (maybeFilepath) {
+            idToOverwrittenFile.set(id, maybeFilepath);
+          }
+        });
+
+        challengeNodes.forEach(node => {
+          idToFilepath.set(node.id, filePath);
+        });
+
+        // we always need to track the created nodes to ensure the pages get
+        // recreated.
+        filePathToCreatedNodes.set(filePath, challengeNodes);
       })
       .catch(e =>
         reporter.error(
@@ -69,12 +164,27 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
     handleChallengeUpdate(filePath, 'added');
   });
 
+  watcher.on('unlink', filePath => {
+    if (!/\.md?$/.test(filePath)) return;
+    handleChallengeUpdate(filePath, 'deleted');
+  });
+
   function sourceAndCreateNodes() {
     return source()
       .then(challenges => Promise.all(challenges))
       .then(challenges => {
         // create challenge nodes
-        challenges.forEach(challenge => createVisibleChallenge(challenge));
+        challenges.forEach(challenge => {
+          const newNode = reportNodeCreationToGatsby(challenge);
+          const existingNodes =
+            filepathToStatefullyCreatedNodes.get(challenge.sourceLocation) ||
+            [];
+          filepathToStatefullyCreatedNodes.set(challenge.sourceLocation, [
+            ...existingNodes,
+            newNode
+          ]);
+          idToFilepath.set(newNode.id, challenge.sourceLocation);
+        });
         // create superblock structure nodes
         createSuperBlockStructureNodes();
         return Promise.resolve();
@@ -89,8 +199,11 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
       });
   }
 
-  function createVisibleChallenge(challenge, options) {
+  function reportNodeCreationToGatsby(challenge, options) {
-    createNode(createChallengeNode(challenge, reporter, options));
+    const challengeNode = createChallengeNode(challenge, reporter, options);
+
+    createNode(challengeNode);
+    return challengeNode;
   }
 
   function createSuperBlockStructureNodes() {
@@ -126,3 +239,118 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
     watcher.on('ready', () => sourceAndCreateNodes().then(resolve, reject));
   });
 };
+
+exports.createPagesStatefully = async function ({ graphql, actions }) {
+  const result = await graphql(`
+    {
+      allChallengeNode(
+        sort: {
+          fields: [
+            challenge___superOrder
+            challenge___order
+            challenge___challengeOrder
+          ]
+        }
+      ) {
+        edges {
+          node {
+            id
+            challenge {
+              block
+              blockLabel
+              blockLayout
+              certification
+              challengeType
+              dashedName
+              demoType
+              disableLoopProtectTests
+              disableLoopProtectPreview
+              fields {
+                slug
+                blockHashSlug
+              }
+              id
+              isLastChallengeInBlock
+              order
+              required {
+                link
+                src
+              }
+              challengeOrder
+              challengeFiles {
+                name
+                ext
+                contents
+                head
+                tail
+                history
+                fileKey
+              }
+              saveSubmissionToDB
+              solutions {
+                contents
+                ext
+                history
+                fileKey
+              }
+              superBlock
+              superOrder
+              template
+              usesMultifileEditor
+              chapter
+              module
+            }
+          }
+        }
+      }
+    }
+  `);
+
+  allChallengeNodes = result.data.allChallengeNode.edges.map(
+    ({ node }) => node
+  );
+
+  const createIdToNextPathMap = nodes =>
+    nodes.reduce((map, node, index) => {
+      const nextNode = nodes[index + 1];
+      const nextPath = nextNode ? nextNode.challenge.fields.slug : null;
+      if (nextPath) map[node.id] = nextPath;
+      return map;
+    }, {});
+
+  const createIdToPrevPathMap = nodes =>
+    nodes.reduce((map, node, index) => {
+      const prevNode = nodes[index - 1];
+      const prevPath = prevNode ? prevNode.challenge.fields.slug : null;
+      if (prevPath) map[node.id] = prevPath;
+      return map;
+    }, {});
+
+  idToNextPathCurrentCurriculum = createIdToNextPathMap(allChallengeNodes);
+
+  idToPrevPathCurrentCurriculum = createIdToPrevPathMap(allChallengeNodes);
+
+  const nodeToPage = createChallengePages(actions.createPage, {
+    idToNextPathCurrentCurriculum,
+    idToPrevPathCurrentCurriculum
+  });
+
+  // Create challenge pages.
+  allChallengeNodes.forEach(nodeToPage);
+};
+
+exports.createPages = function ({ actions }) {
+  // actions.createPage has to be called in the createPages hook
+  const nodes = [...filePathToCreatedNodes.values()].flat();
+  for (const node of nodes) {
+    const nodeToPage = createChallengePages(actions.createPage, {
+      idToNextPathCurrentCurriculum,
+      idToPrevPathCurrentCurriculum
+    });
+
+    nodeToPage(node, 0, allChallengeNodes);
+  }
+
+  // It's important NOT to clear the createdNodes, since Gatsby deletes any
+  // pages that are not recreated each time createPages is called.
+};