refactor(client): speed up client updates (#65025)

tools/client-plugins/gatsby-source-challenges/gatsby-node.js

@@ -1,4 +1,5 @@
 const chokidar = require('chokidar');
+
 const {
   getSuperblockStructure
 } = require('@freecodecamp/curriculum/file-handler');
@@ -7,6 +8,23 @@ const {
 } = require('@freecodecamp/curriculum/build-curriculum');
 
 const { createChallengeNode } = require('./create-challenge-nodes');
+const {
+  createChallengePages,
+  getTemplateComponent
+} = require('../../../client/utils/gatsby');
+
+// createPagesStatefully only runs once, but we need the following when
+// updating challenges, so they have to be stored in memory.
+let allChallengeNodes;
+let idToNextPathCurrentCurriculum;
+let idToPrevPathCurrentCurriculum;
+const filepathToStatefullyCreatedNodes = new Map();
+const filePathToCreatedNodes = new Map();
+// reverse lookup, to detect if an updated file has "overwritten" another file
+// (i.e. the updated file now has the same node id as another file).
+const idToFilepath = new Map();
+// recently overwritten files
+const idToOverwrittenFile = new Map();
 
 exports.sourceNodes = function sourceChallengesSourceNodes(
   { actions, reporter, createNodeId, createContentDigest },
@@ -31,7 +49,7 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
     a path to a curriculum directory
     `);
   }
-  const { createNode } = actions;
+  const { createNode, deleteNode, deletePage } = actions;
   const watcher = chokidar.watch(curriculumPath, {
     ignored: /(^|[/\\])\../,
     ignoreInitial: true,
@@ -39,16 +57,93 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
     cwd: curriculumPath
   });
 
+  function deletePages(filePath) {
+    const statefulNodes = filepathToStatefullyCreatedNodes.get(filePath) || [];
+    statefulNodes.forEach(node => {
+      deleteNode(node);
+      deletePage({
+        path: node.challenge.fields.slug,
+        component: getTemplateComponent(node.challenge.challengeType)
+      });
+      idToFilepath.delete(node.id);
+    });
+
+    const createdNodes = filePathToCreatedNodes.get(filePath) || [];
+    createdNodes.forEach(node => {
+      deleteNode(node);
+      idToFilepath.delete(node.id);
+    });
+
+    filepathToStatefullyCreatedNodes.delete(filePath);
+    filePathToCreatedNodes.delete(filePath);
+  }
+
+  function tryToDeletePages(filePath) {
+    const oldCreatedNodeIds = (filePathToCreatedNodes.get(filePath) ?? []).map(
+      node => node.id
+    );
+
+    const oldStatefullyCreatedNodeIds = (
+      filepathToStatefullyCreatedNodes.get(filePath) ?? []
+    ).map(node => node.id);
+
+    const oldNodeIds = [...oldCreatedNodeIds, ...oldStatefullyCreatedNodeIds];
+
+    const overwrittenFiles = new Set(
+      oldNodeIds.map(id => idToOverwrittenFile.get(id))
+    );
+
+    if (overwrittenFiles.has(filePath)) {
+      // since this has already been overwritten, it doesn't need
+      // deleting, but there's no longer any need to track that it was
+      // overwritten.
+      oldNodeIds.forEach(id => {
+        idToOverwrittenFile.delete(id);
+      });
+    } else {
+      deletePages(filePath);
+    }
+  }
+
   function handleChallengeUpdate(filePath, action = 'changed') {
+    if (action === 'deleted') {
+      // We have to return before calling onSourceChange, since the file is
+      // gone.
+      return tryToDeletePages(filePath);
+    }
+
     return onSourceChange(filePath)
       .then(challenges => {
         const actionText = action === 'added' ? 'creating' : 'replacing';
         reporter.info(
           `Challenge file ${action}: ${filePath}, ${actionText} challengeNodes with ids ${challenges.map(({ id }) => id).join(', ')}`
         );
-        challenges.forEach(challenge =>
-          createVisibleChallenge(challenge, { isReloading: true })
+
+        if (action === 'changed') {
+          tryToDeletePages(filePath);
+        }
+
+        const challengeNodes = challenges.map(challenge =>
+          reportNodeCreationToGatsby(challenge, {
+            isReloading: true
+          })
         );
+
+        // Track if file has been overwritten.
+        challengeNodes.forEach(({ id }) => {
+          const maybeFilepath = idToFilepath.get(id);
+          if (maybeFilepath) {
+            idToOverwrittenFile.set(id, maybeFilepath);
+          }
+        });
+
+        challengeNodes.forEach(node => {
+          idToFilepath.set(node.id, filePath);
+        });
+
+        // we always need to track the created nodes to ensure the pages get
+        // recreated.
+        filePathToCreatedNodes.set(filePath, challengeNodes);
       })
       .catch(e =>
         reporter.error(
@@ -69,12 +164,27 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
     handleChallengeUpdate(filePath, 'added');
   });
 
+  watcher.on('unlink', filePath => {
+    if (!/\.md?$/.test(filePath)) return;
+    handleChallengeUpdate(filePath, 'deleted');
+  });
+
   function sourceAndCreateNodes() {
     return source()
       .then(challenges => Promise.all(challenges))
       .then(challenges => {
         // create challenge nodes
-        challenges.forEach(challenge => createVisibleChallenge(challenge));
+        challenges.forEach(challenge => {
+          const newNode = reportNodeCreationToGatsby(challenge);
+          const existingNodes =
+            filepathToStatefullyCreatedNodes.get(challenge.sourceLocation) ||
+            [];
+          filepathToStatefullyCreatedNodes.set(challenge.sourceLocation, [
+            ...existingNodes,
+            newNode
+          ]);
+          idToFilepath.set(newNode.id, challenge.sourceLocation);
+        });
         // create superblock structure nodes
         createSuperBlockStructureNodes();
         return Promise.resolve();
@@ -89,8 +199,11 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
       });
   }
 
-  function createVisibleChallenge(challenge, options) {
-    createNode(createChallengeNode(challenge, reporter, options));
+  function reportNodeCreationToGatsby(challenge, options) {
+    const challengeNode = createChallengeNode(challenge, reporter, options);
+
+    createNode(challengeNode);
+    return challengeNode;
   }
 
   function createSuperBlockStructureNodes() {
@@ -126,3 +239,118 @@ exports.sourceNodes = function sourceChallengesSourceNodes(
     watcher.on('ready', () => sourceAndCreateNodes().then(resolve, reject));
   });
 };
+
+exports.createPagesStatefully = async function ({ graphql, actions }) {
+  const result = await graphql(`
+    {
+      allChallengeNode(
+        sort: {
+          fields: [
+            challenge___superOrder
+            challenge___order
+            challenge___challengeOrder
+          ]
+        }
+      ) {
+        edges {
+          node {
+            id
+            challenge {
+              block
+              blockLabel
+              blockLayout
+              certification
+              challengeType
+              dashedName
+              demoType
+              disableLoopProtectTests
+              disableLoopProtectPreview
+              fields {
+                slug
+                blockHashSlug
+              }
+              id
+              isLastChallengeInBlock
+              order
+              required {
+                link
+                src
+              }
+              challengeOrder
+              challengeFiles {
+                name
+                ext
+                contents
+                head
+                tail
+                history
+                fileKey
+              }
+              saveSubmissionToDB
+              solutions {
+                contents
+                ext
+                history
+                fileKey
+              }
+              superBlock
+              superOrder
+              template
+              usesMultifileEditor
+              chapter
+              module
+            }
+          }
+        }
+      }
+    }
+  `);
+
+  allChallengeNodes = result.data.allChallengeNode.edges.map(
+    ({ node }) => node
+  );
+
+  const createIdToNextPathMap = nodes =>
+    nodes.reduce((map, node, index) => {
+      const nextNode = nodes[index + 1];
+      const nextPath = nextNode ? nextNode.challenge.fields.slug : null;
+      if (nextPath) map[node.id] = nextPath;
+      return map;
+    }, {});
+
+  const createIdToPrevPathMap = nodes =>
+    nodes.reduce((map, node, index) => {
+      const prevNode = nodes[index - 1];
+      const prevPath = prevNode ? prevNode.challenge.fields.slug : null;
+      if (prevPath) map[node.id] = prevPath;
+      return map;
+    }, {});
+
+  idToNextPathCurrentCurriculum = createIdToNextPathMap(allChallengeNodes);
+
+  idToPrevPathCurrentCurriculum = createIdToPrevPathMap(allChallengeNodes);
+
+  const nodeToPage = createChallengePages(actions.createPage, {
+    idToNextPathCurrentCurriculum,
+    idToPrevPathCurrentCurriculum
+  });
+
+  // Create challenge pages.
+  allChallengeNodes.forEach(nodeToPage);
+};
+
+exports.createPages = function ({ actions }) {
+  // actions.createPage has to be called in the createPages hook
+  const nodes = [...filePathToCreatedNodes.values()].flat();
+  for (const node of nodes) {
+    const nodeToPage = createChallengePages(actions.createPage, {
+      idToNextPathCurrentCurriculum,
+      idToPrevPathCurrentCurriculum
+    });
+
+    nodeToPage(node, 0, allChallengeNodes);
+  }
+
+  // It's important NOT to clear the createdNodes, since Gatsby deletes any
+  // pages that are not recreated each time createPages is called.
+};