Understanding the packaging principle of js from mini-pack

The main content of this article is derived from mini-pack. The example code differs slightly from the original project's code.

When developing a web app, a modular development approach is generally adopted, meaning that different business requirements and components are written into different files, which are then merged into one or more files as needed. This process is called bundling.

The above image shows a common file structure, where the entry file (entry.js) depends on a.js and c.js, a.js depends on b.js, and c.js depends on d.js and e.js. Of course, the actual structure can be more complex than the example. If we need to bundle such a code organization structure, then all files should be bundled into one file.

First, we need to analyze the dependency relationships of each file:

entry.js depends on a.js and c.js
a.js depends on b.js
c.js depends on d.js and e.js

In the code, we can represent this roughly as follows:

{
  "entry.js": {
    "id": 0,
    "code": "xxx",
    "deps": ["a.js", "c.js"]
  },
  "a.js": {
    "id": 1,
    "code": "xxx",
    "deps": ["b.js"]
  },
  "c.js": {
    "id": 2,
    "code": "xxx",
    "deps": ["d.js", "f.js"]
  },
  "b.js": {
    "id": 3,
    "code": "xxx",
    "deps": []
  },
  "d.js": {
    "id": 4,
    "code": "xxx",
    "deps": []
  },
  "e.js": {
    "id": 0,
    "code": "xxx",
    "deps": []
  }
}

Each file has its own id, code, and deps. During the bundling process, we can start from the entry file to transpile the code (considering compatibility), then use id and deps to find other files to perform the same operation, and finally merge all files into a single file in string format, thus completing the bundling.

Next, let's start from a practical scenario:

# Project File Structure

- `mini-bundle`
  - `index.js`
    - `example/`
	    - `entry.js`
	    - `hello.js`
	    - `name.js`

entry.js

import { sayHello } from "./hello.js";

console.log(sayHello());

hello.js

import { name } from "./name.js";

export function sayHello() {
  return `hello ${name}`;
}

name.js

export const name = "mini-bundle";

First, we need to generate an asset for each js file. An asset can be understood as the data structure corresponding to each js file for bundling.

id is the unique identifier of the file, used to query the compiled module;

filename is the relative path of the file (it can also be an absolute path or alias), used to find, read, and compile the file content;

dependencies are the dependencies of the file;

code is the compiled code;

mapping is the mapping of filename to id, which can be used with dependencies to find the id of dependencies;

Therefore, we need to implement a createAsset function first. The parse and traverse functions in the code are from Babel, used to parse and generate AST and traverse to query dependencies.

async function createAsset(filename) {
  // Read file content
  const content = await readFile(filename, "utf-8");

  // Build ast
  const ast = parse(content, {
    sourceType: "module",
  });

  // Collect dependencies
  const dependencies = [];
  traverse.default(ast, {
    ImportDeclaration: ({ node }) => {
      dependencies.push(node.source.value);
    },
  });

  const id = ID++;

  // Transpile code
  const { code } = transformFromAst(ast, null, {
    presets: ["@babel/preset-env"],
  });

	// The returned object currently does not have a mapping property
	// This property will be created when building the dependency graph
  return {
    id,
    filename,
    dependencies,
    code,
  };
}

Next, we need to build the entire dependency graph (the dependencies of all files) starting from the entry file.

async function createGraph(entry) {
  const mainAsset = await createAsset(entry);

  // Record all module assets
  const queue = [mainAsset];

  // Breadth-first traversal
  for (const asset of queue) {
    // Record the module's dependency relationships
    // Map module id and module path
    asset.mapping = {};
    // Get the module's directory
    const dirname = path.dirname(asset.filename);

    for (const relativePath of asset.dependencies) {
      // Get absolute path
      const absolutePath = path.join(dirname, relativePath);
      // Create child module
      const child = await createAsset(absolutePath);
      // Record the module's dependency relationships
      asset.mapping[relativePath] = child.id;
      // Add child module to the queue
      queue.push(child);
    }
  }

  return queue;
}

// const graph = await createGraph("./example/entry.js");

When building the dependency graph, it starts from the entry file (entry.js) to generate the mainAsset, then adds the mainAsset to the queue, traverses the queue to find the files corresponding to dependencies, and sequentially generates child assets of dependencies and updates the mapping, then adds the child to the queue until all files are generated.

Finally, the bundle function concatenates all assets.

function bundle(graph) {
  let modules = "";

  graph.forEach(mod => {
    modules += `${mod.id}: [
      function(require, module, exports) {
        ${mod.code}
      },
      ${JSON.stringify(mod.mapping)}
    ],
    `;
  });

  return `
    (function(modules) {
      function require(id) {
        const [fn, mapping] = modules[id];
        function localRequire(name) {
          return require(mapping[name]);
        }
        const module = { exports: {} };
        fn(localRequire, module, module.exports);
        return module.exports;
      }
      require(0);
    })({${modules}})
  `;
}

In the bundle function, the graph is first converted into a modules object structured as { id: [fn, mapping] }, and then passed into an immediately invoked function (according to the CommonJS specification), thus completing the script bundling.

Here is the complete code:

import { readFile } from "node:fs/promises";
import { parse } from "@babel/parser";
import traverse from "@babel/traverse";
import { transformFromAst } from "@babel/core";
import path from "node:path";
import { mkdirSync, writeFileSync } from "node:fs";
let ID = 0;

async function createAsset(filename) {
  // Read file content
  const content = await readFile(filename, "utf-8");

  // Build ast
  const ast = parse(content, {
    sourceType: "module",
  });

  // Collect dependencies
  const dependencies = [];
  traverse.default(ast, {
    ImportDeclaration: ({ node }) => {
      dependencies.push(node.source.value);
    },
  });

  const id = ID++;

  // Transpile code
  const { code } = transformFromAst(ast, null, {
    presets: ["@babel/preset-env"],
  });

  return {
    id,
    filename,
    dependencies,
    code,
  };
}

async function createGraph(entry) {
  const mainAsset = await createAsset(entry);

  // Record all module assets
  const queue = [mainAsset];

  // Breadth-first traversal
  for (const asset of queue) {
    // Record the module's dependency relationships
    // Map module id and module path
    asset.mapping = {};
    // Get the module's directory
    const dirname = path.dirname(asset.filename);

    for (const relativePath of asset.dependencies) {
      // Get absolute path
      const absolutePath = path.join(dirname, relativePath);
      // Create child module
      const child = await createAsset(absolutePath);
      // Record the module's dependency relationships
      asset.mapping[relativePath] = child.id;
      // Add child module to the queue
      queue.push(child);
    }
  }

  return queue;
}

function bundle(graph) {
  let modules = "";

  graph.forEach(mod => {
    modules += `${mod.id}: [
      function(require, module, exports) {
        ${mod.code}
      },
      ${JSON.stringify(mod.mapping)}
    ],
    `;
  });

  return `
    (function(modules) {
      function require(id) {
        const [fn, mapping] = modules[id];
        function localRequire(name) {
          return require(mapping[name]);
        }
        const module = { exports: {} };
        fn(localRequire, module, module.exports);
        return module.exports;
      }
      require(0);
    })({${modules}})
  `;
}

(async () => {
  const graph = await createGraph("./example/entry.js");
  const bundleCode = bundle(graph);

  mkdirSync("./dist", { recursive: true });
  writeFileSync("./dist/bundle.js", bundleCode);
})();

The code after bundling entry.js, hello.js, and name.js is as follows:


(function (modules) {
  function require(id) {
    const [fn, mapping] = modules[id];
    function localRequire(name) {
      return require(mapping[name]);
    }
    const module = { exports: {} };
    fn(localRequire, module, module.exports);
    return module.exports;
  }
  require(0);
})({
  0: [
    function (require, module, exports) {
      "use strict";

      var _hello = require("./hello.js");
      console.log((0, _hello.sayHello)());
    },
    { "./hello.js": 1 }
  ],
  1: [
    function (require, module, exports) {
      "use strict";

      Object.defineProperty(exports, "__esModule", {
        value: true
      });
      exports.sayHello = sayHello;
      var _name = require("./name.js");
      function sayHello() {
        return "hello ".concat(_name.name);
      }
    },
    { "./name.js": 2 }
  ],
  2: [
    function (require, module, exports) {
      "use strict";

      Object.defineProperty(exports, "__esModule", {
        value: true
      });
      exports.name = void 0;
      var name = exports.name = "mini-bundle";
    },
    {}
  ],
})

The content of this article covers the basic principles of bundling, mainly focusing on understanding the concepts of asset and graph and how to find dependencies. Other issues such as caching and circular references are not addressed.