From e6eb338fe65718ce23b890f3ae97b9007dee96af Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Tue, 10 Feb 2026 13:39:55 +0100 Subject: [PATCH] refactor(client): speed up client updates (#65025) --- client/gatsby-node.js | 88 +------ client/utils/build-challenges.js | 1 - client/utils/gatsby/challenge-page-creator.js | 24 +- curriculum/schema/challenge-schema.js | 2 + curriculum/src/build-superblock.ts | 9 +- .../gatsby-source-challenges/gatsby-node.js | 240 +++++++++++++++++- 6 files changed, 258 insertions(+), 106 deletions(-) diff --git a/client/gatsby-node.js b/client/gatsby-node.js index 1e164f20a10..2a3f760bd4d 100644 --- a/client/gatsby-node.js +++ b/client/gatsby-node.js @@ -8,7 +8,6 @@ const webpack = require('webpack'); const { SuperBlocks } = require('@freecodecamp/shared/config/curriculum'); const env = require('./config/env.json'); const { - createChallengePages, createBlockIntroPages, createSuperBlockIntroPages } = require('./utils/gatsby'); @@ -60,62 +59,11 @@ exports.createPages = async function createPages({ const result = await graphql(` { - allChallengeNode( - sort: { - fields: [ - challenge___superOrder - challenge___order - challenge___challengeOrder - ] - } - ) { + allChallengeNode { edges { node { - id challenge { block - blockLabel - blockLayout - certification - challengeType - dashedName - demoType - disableLoopProtectTests - disableLoopProtectPreview - fields { - slug - blockHashSlug - } - id - isLastChallengeInBlock - order - required { - link - src - } - challengeOrder - challengeFiles { - name - ext - contents - head - tail - history - fileKey - } - saveSubmissionToDB - solutions { - contents - ext - history - fileKey - } - superBlock - superOrder - template - usesMultifileEditor - chapter - module } } } @@ -140,40 +88,6 @@ exports.createPages = async function createPages({ } `); - const allChallengeNodes = result.data.allChallengeNode.edges.map( - ({ node }) => node - ); - - const createIdToNextPathMap = nodes => - nodes.reduce((map, node, index) => { - const nextNode = nodes[index + 1]; - const nextPath = nextNode ? nextNode.challenge.fields.slug : null; - if (nextPath) map[node.id] = nextPath; - return map; - }, {}); - - const createIdToPrevPathMap = nodes => - nodes.reduce((map, node, index) => { - const prevNode = nodes[index - 1]; - const prevPath = prevNode ? prevNode.challenge.fields.slug : null; - if (prevPath) map[node.id] = prevPath; - return map; - }, {}); - - const idToNextPathCurrentCurriculum = - createIdToNextPathMap(allChallengeNodes); - - const idToPrevPathCurrentCurriculum = - createIdToPrevPathMap(allChallengeNodes); - - // Create challenge pages. - result.data.allChallengeNode.edges.forEach( - createChallengePages(createPage, { - idToNextPathCurrentCurriculum, - idToPrevPathCurrentCurriculum - }) - ); - const blocks = uniq( result.data.allChallengeNode.edges.map( ({ diff --git a/client/utils/build-challenges.js b/client/utils/build-challenges.js index 5c032ba7859..e09c8d61467 100644 --- a/client/utils/build-challenges.js +++ b/client/utils/build-challenges.js @@ -51,7 +51,6 @@ exports.replaceChallengeNodes = () => { const block = path.basename(parentDir); const filename = path.basename(filePath); - console.log(`Replacing challenge nodes for ${filePath}`); const meta = getBlockStructure(block); const superblocks = getSuperblocks(block); diff --git a/client/utils/gatsby/challenge-page-creator.js b/client/utils/gatsby/challenge-page-creator.js index eccfd85e56d..62805d5dcc6 100644 --- a/client/utils/gatsby/challenge-page-creator.js +++ b/client/utils/gatsby/challenge-page-creator.js @@ -69,23 +69,25 @@ const views = { examDownload }; -function getIsFirstStepInBlock(id, edges) { - const current = edges[id]; - const previous = edges[id - 1]; +function getIsFirstStepInBlock(id, nodes) { + const current = nodes[id]; + const previous = nodes[id - 1]; if (!previous) return true; - return previous.node.challenge.block !== current.node.challenge.block; + return previous.challenge.block !== current.challenge.block; } function getTemplateComponent(challengeType) { return views[viewTypes[challengeType]]; } +exports.getTemplateComponent = getTemplateComponent; + exports.createChallengePages = function ( createPage, { idToNextPathCurrentCurriculum, idToPrevPathCurrentCurriculum } ) { - return function ({ node }, index, allChallengeEdges) { + return function (node, index, allChallengeNodes) { const { dashedName, disableLoopProtectTests, @@ -118,7 +120,7 @@ exports.createChallengePages = function ( chapter, module, block, - isFirstStep: getIsFirstStepInBlock(index, allChallengeEdges), + isFirstStep: getIsFirstStepInBlock(index, allChallengeNodes), template, required, isLastChallengeInBlock: isLastChallengeInBlock, @@ -129,7 +131,7 @@ exports.createChallengePages = function ( }, projectPreview: getProjectPreviewConfig( node.challenge, - allChallengeEdges + allChallengeNodes ), id: node.id } @@ -140,12 +142,12 @@ exports.createChallengePages = function ( // TODO: figure out a cleaner way to get the last challenge in a block. Create // it during the curriculum build process and attach it to the first challenge? // That would remove the need to analyse allChallengeEdges. -function getProjectPreviewConfig(challenge, allChallengeEdges) { +function getProjectPreviewConfig(challenge, allChallengeNodes) { const { block } = challenge; - const challengesInBlock = allChallengeEdges - .filter(({ node: { challenge } }) => challenge.block === block) - .map(({ node: { challenge } }) => challenge); + const challengesInBlock = allChallengeNodes + .filter(({ challenge }) => challenge.block === block) + .map(({ challenge }) => challenge); const lastChallenge = challengesInBlock[challengesInBlock.length - 1]; const solutionFiles = lastChallenge.solutions[0] ?? []; const lastChallengeFiles = lastChallenge.challengeFiles ?? []; diff --git a/curriculum/schema/challenge-schema.js b/curriculum/schema/challenge-schema.js index 0abc7d5b270..8c0f1f3178e 100644 --- a/curriculum/schema/challenge-schema.js +++ b/curriculum/schema/challenge-schema.js @@ -342,6 +342,8 @@ const schema = Joi.object().keys({ }) }), showSpeakingButton: Joi.bool(), + // This is only to be used for dynamic client updates. + sourceLocation: Joi.string(), solutions: Joi.array().items(Joi.array().items(fileJoi).min(1)), superBlock: Joi.string().regex(slugWithSlashRE), superOrder: Joi.number(), diff --git a/curriculum/src/build-superblock.ts b/curriculum/src/build-superblock.ts index 5d6e48c6e75..665bee774d7 100644 --- a/curriculum/src/build-superblock.ts +++ b/curriculum/src/build-superblock.ts @@ -1,5 +1,5 @@ import { existsSync, readdirSync } from 'fs'; -import { resolve } from 'path'; +import { join, resolve, basename } from 'path'; import { isEmpty } from 'lodash'; import debug from 'debug'; @@ -336,6 +336,13 @@ export class BlockCreator { ); challenge.translationPending = this.lang !== 'english' && !isAudited; + // Add source location to allow tracing back to original file (necessary to + // update the client when files change) + challenge.sourceLocation = join( + basename(this.blockContentDir), + block, + filename + ); return finalizeChallenge(challenge, meta); } diff --git a/tools/client-plugins/gatsby-source-challenges/gatsby-node.js b/tools/client-plugins/gatsby-source-challenges/gatsby-node.js index 78d80dab413..a4722725ef5 100644 --- a/tools/client-plugins/gatsby-source-challenges/gatsby-node.js +++ b/tools/client-plugins/gatsby-source-challenges/gatsby-node.js @@ -1,4 +1,5 @@ const chokidar = require('chokidar'); + const { getSuperblockStructure } = require('@freecodecamp/curriculum/file-handler'); @@ -7,6 +8,23 @@ const { } = require('@freecodecamp/curriculum/build-curriculum'); const { createChallengeNode } = require('./create-challenge-nodes'); +const { + createChallengePages, + getTemplateComponent +} = require('../../../client/utils/gatsby'); + +// createPagesStatefully only runs once, but we need the following when +// updating challenges, so they have to be stored in memory. +let allChallengeNodes; +let idToNextPathCurrentCurriculum; +let idToPrevPathCurrentCurriculum; +const filepathToStatefullyCreatedNodes = new Map(); +const filePathToCreatedNodes = new Map(); +// reverse lookup, to detect if an updated file has "overwritten" another file +// (i.e. the updated file now has the same node id as another file). +const idToFilepath = new Map(); +// recently overwritten files +const idToOverwrittenFile = new Map(); exports.sourceNodes = function sourceChallengesSourceNodes( { actions, reporter, createNodeId, createContentDigest }, @@ -31,7 +49,7 @@ exports.sourceNodes = function sourceChallengesSourceNodes( a path to a curriculum directory `); } - const { createNode } = actions; + const { createNode, deleteNode, deletePage } = actions; const watcher = chokidar.watch(curriculumPath, { ignored: /(^|[/\\])\../, ignoreInitial: true, @@ -39,16 +57,93 @@ exports.sourceNodes = function sourceChallengesSourceNodes( cwd: curriculumPath }); + function deletePages(filePath) { + const statefulNodes = filepathToStatefullyCreatedNodes.get(filePath) || []; + statefulNodes.forEach(node => { + deleteNode(node); + deletePage({ + path: node.challenge.fields.slug, + component: getTemplateComponent(node.challenge.challengeType) + }); + idToFilepath.delete(node.id); + }); + + const createdNodes = filePathToCreatedNodes.get(filePath) || []; + createdNodes.forEach(node => { + deleteNode(node); + idToFilepath.delete(node.id); + }); + + filepathToStatefullyCreatedNodes.delete(filePath); + filePathToCreatedNodes.delete(filePath); + } + + function tryToDeletePages(filePath) { + const oldCreatedNodeIds = (filePathToCreatedNodes.get(filePath) ?? []).map( + node => node.id + ); + + const oldStatefullyCreatedNodeIds = ( + filepathToStatefullyCreatedNodes.get(filePath) ?? [] + ).map(node => node.id); + + const oldNodeIds = [...oldCreatedNodeIds, ...oldStatefullyCreatedNodeIds]; + + const overwrittenFiles = new Set( + oldNodeIds.map(id => idToOverwrittenFile.get(id)) + ); + + if (overwrittenFiles.has(filePath)) { + // since this has already been overwritten, it doesn't need + // deleting, but there's no longer any need to track that it was + // overwritten. + oldNodeIds.forEach(id => { + idToOverwrittenFile.delete(id); + }); + } else { + deletePages(filePath); + } + } + function handleChallengeUpdate(filePath, action = 'changed') { + if (action === 'deleted') { + // We have to return before calling onSourceChange, since the file is + // gone. + return tryToDeletePages(filePath); + } + return onSourceChange(filePath) .then(challenges => { const actionText = action === 'added' ? 'creating' : 'replacing'; reporter.info( `Challenge file ${action}: ${filePath}, ${actionText} challengeNodes with ids ${challenges.map(({ id }) => id).join(', ')}` ); - challenges.forEach(challenge => - createVisibleChallenge(challenge, { isReloading: true }) + + if (action === 'changed') { + tryToDeletePages(filePath); + } + + const challengeNodes = challenges.map(challenge => + reportNodeCreationToGatsby(challenge, { + isReloading: true + }) ); + + // Track if file has been overwritten. + challengeNodes.forEach(({ id }) => { + const maybeFilepath = idToFilepath.get(id); + if (maybeFilepath) { + idToOverwrittenFile.set(id, maybeFilepath); + } + }); + + challengeNodes.forEach(node => { + idToFilepath.set(node.id, filePath); + }); + + // we always need to track the created nodes to ensure the pages get + // recreated. + filePathToCreatedNodes.set(filePath, challengeNodes); }) .catch(e => reporter.error( @@ -69,12 +164,27 @@ exports.sourceNodes = function sourceChallengesSourceNodes( handleChallengeUpdate(filePath, 'added'); }); + watcher.on('unlink', filePath => { + if (!/\.md?$/.test(filePath)) return; + handleChallengeUpdate(filePath, 'deleted'); + }); + function sourceAndCreateNodes() { return source() .then(challenges => Promise.all(challenges)) .then(challenges => { // create challenge nodes - challenges.forEach(challenge => createVisibleChallenge(challenge)); + challenges.forEach(challenge => { + const newNode = reportNodeCreationToGatsby(challenge); + const existingNodes = + filepathToStatefullyCreatedNodes.get(challenge.sourceLocation) || + []; + filepathToStatefullyCreatedNodes.set(challenge.sourceLocation, [ + ...existingNodes, + newNode + ]); + idToFilepath.set(newNode.id, challenge.sourceLocation); + }); // create superblock structure nodes createSuperBlockStructureNodes(); return Promise.resolve(); @@ -89,8 +199,11 @@ exports.sourceNodes = function sourceChallengesSourceNodes( }); } - function createVisibleChallenge(challenge, options) { - createNode(createChallengeNode(challenge, reporter, options)); + function reportNodeCreationToGatsby(challenge, options) { + const challengeNode = createChallengeNode(challenge, reporter, options); + + createNode(challengeNode); + return challengeNode; } function createSuperBlockStructureNodes() { @@ -126,3 +239,118 @@ exports.sourceNodes = function sourceChallengesSourceNodes( watcher.on('ready', () => sourceAndCreateNodes().then(resolve, reject)); }); }; + +exports.createPagesStatefully = async function ({ graphql, actions }) { + const result = await graphql(` + { + allChallengeNode( + sort: { + fields: [ + challenge___superOrder + challenge___order + challenge___challengeOrder + ] + } + ) { + edges { + node { + id + challenge { + block + blockLabel + blockLayout + certification + challengeType + dashedName + demoType + disableLoopProtectTests + disableLoopProtectPreview + fields { + slug + blockHashSlug + } + id + isLastChallengeInBlock + order + required { + link + src + } + challengeOrder + challengeFiles { + name + ext + contents + head + tail + history + fileKey + } + saveSubmissionToDB + solutions { + contents + ext + history + fileKey + } + superBlock + superOrder + template + usesMultifileEditor + chapter + module + } + } + } + } + } + `); + + allChallengeNodes = result.data.allChallengeNode.edges.map( + ({ node }) => node + ); + + const createIdToNextPathMap = nodes => + nodes.reduce((map, node, index) => { + const nextNode = nodes[index + 1]; + const nextPath = nextNode ? nextNode.challenge.fields.slug : null; + if (nextPath) map[node.id] = nextPath; + return map; + }, {}); + + const createIdToPrevPathMap = nodes => + nodes.reduce((map, node, index) => { + const prevNode = nodes[index - 1]; + const prevPath = prevNode ? prevNode.challenge.fields.slug : null; + if (prevPath) map[node.id] = prevPath; + return map; + }, {}); + + idToNextPathCurrentCurriculum = createIdToNextPathMap(allChallengeNodes); + + idToPrevPathCurrentCurriculum = createIdToPrevPathMap(allChallengeNodes); + + const nodeToPage = createChallengePages(actions.createPage, { + idToNextPathCurrentCurriculum, + idToPrevPathCurrentCurriculum + }); + + // Create challenge pages. + allChallengeNodes.forEach(nodeToPage); +}; + +exports.createPages = function ({ actions }) { + // actions.createPage has to be called in the createPages hook + const nodes = [...filePathToCreatedNodes.values()].flat(); + for (const node of nodes) { + const nodeToPage = createChallengePages(actions.createPage, { + idToNextPathCurrentCurriculum, + idToPrevPathCurrentCurriculum + }); + + nodeToPage(node, 0, allChallengeNodes); + } + + // It's important NOT to clear the createdNodes, since Gatsby deletes any + // pages that are not recreated each time createPages is called. +};