525 lines
18 KiB
JavaScript
525 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* find-duplicates — component duplication scanner for the web-app source.
|
||
*
|
||
* Parses every TSX file under `src` with the TypeScript compiler API and
|
||
* computes four things:
|
||
*
|
||
* 1. Layout idiom census — exact VStack/HStack/Card/Section/Grid prop
|
||
* configurations repeated across the codebase (e.g. `VStack(gap=0)`).
|
||
* 2. Structural fingerprints — normalized JSX subtree shapes (text and
|
||
* non-layout prop values stripped) that recur in ≥ N places. These are
|
||
* the raw material for extracting shared components.
|
||
* 3. Known idioms — hand-tuned recognizers for patterns that keep showing
|
||
* up by hand: label+description pairs, stat cards, section headers,
|
||
* icon+text rows, form action footers, wizard progress steps.
|
||
* 4. File similarity — pairwise Jaccard over each file's structural
|
||
* fingerprints, surfacing pages that are near-copies of each other.
|
||
*
|
||
* Usage:
|
||
* node scripts/find-duplicates.mjs # run from apps/web-app
|
||
* node scripts/find-duplicates.mjs --root src --min-count 3
|
||
* node scripts/find-duplicates.mjs --json > report.json
|
||
* node scripts/find-duplicates.mjs --limit 8 # rows per section
|
||
*
|
||
* Zero runtime dependencies: only `typescript` (already a devDependency).
|
||
*/
|
||
import { readdirSync, readFileSync, statSync } from "node:fs";
|
||
import { join, relative, resolve } from "node:path";
|
||
import { createRequire } from "node:module";
|
||
|
||
const require = createRequire(import.meta.url);
|
||
const ts = require("typescript");
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// CLI args
|
||
// ---------------------------------------------------------------------------
|
||
const args = process.argv.slice(2);
|
||
const flag = (name, fallback) => {
|
||
const i = args.indexOf(name);
|
||
return i === -1 ? fallback : args[i + 1];
|
||
};
|
||
const has = (name) => args.includes(name);
|
||
|
||
const ROOT = flag("--root", "src");
|
||
const MIN_COUNT = Number(flag("--min-count", "3"));
|
||
const LIMIT = Number(flag("--limit", "10"));
|
||
const AS_JSON = has("--json");
|
||
|
||
const LAYOUT_TAGS = new Set([
|
||
"VStack",
|
||
"HStack",
|
||
"StackItem",
|
||
"Section",
|
||
"Card",
|
||
"Grid",
|
||
"Center",
|
||
"Layout",
|
||
"LayoutHeader",
|
||
"LayoutContent",
|
||
"LayoutFooter",
|
||
"LayoutPanel",
|
||
]);
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Source collection
|
||
// ---------------------------------------------------------------------------
|
||
const SRC_DIR = resolve(process.cwd(), ROOT);
|
||
if (!statSync(SRC_DIR, { throwIfNoParent: false })?.isDirectory?.()) {
|
||
console.error(`Cannot find source directory: ${SRC_DIR}`);
|
||
process.exit(1);
|
||
}
|
||
|
||
/** Collect all .tsx files under a directory, skipping build/scratch dirs. */
|
||
function collectTsx(dir, out = []) {
|
||
for (const entry of readdirSync(dir)) {
|
||
if (entry === "node_modules" || entry === "dist" || entry === ".astryx-tmp") continue;
|
||
const full = join(dir, entry);
|
||
const st = statSync(full);
|
||
if (st.isDirectory()) collectTsx(full, out);
|
||
else if (full.endsWith(".tsx") || full.endsWith(".ts")) out.push(full);
|
||
}
|
||
return out;
|
||
}
|
||
|
||
const files = collectTsx(SRC_DIR).sort();
|
||
const parsed = [];
|
||
|
||
for (const file of files) {
|
||
const text = readFileSync(file, "utf8");
|
||
const sf = ts.createSourceFile(file, text, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX);
|
||
parsed.push({ file, text, sf });
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// AST helpers
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/** Normalized attributes of a JSX element: [{key, value}] with literals resolved. */
|
||
function attrsOf(node) {
|
||
const out = [];
|
||
for (const prop of node.attributes.properties) {
|
||
if (!ts.isJsxAttribute(prop)) {
|
||
out.push({ key: "...", value: "..." }); // spread
|
||
continue;
|
||
}
|
||
const key = prop.name.getText();
|
||
const init = prop.initializer;
|
||
if (!init) {
|
||
out.push({ key, value: "true" }); // boolean shorthand
|
||
} else if (ts.isStringLiteral(init)) {
|
||
out.push({ key, value: init.text });
|
||
} else if (ts.isJsxExpression(init)) {
|
||
const e = init.expression;
|
||
if (!e) out.push({ key, value: "expr" });
|
||
else if (ts.isStringLiteral(e) || ts.isNumericLiteral(e)) out.push({ key, value: e.text });
|
||
else if (e.kind === ts.SyntaxKind.TrueKeyword) out.push({ key, value: "true" });
|
||
else if (e.kind === ts.SyntaxKind.FalseKeyword) out.push({ key, value: "false" });
|
||
else out.push({ key, value: "expr" });
|
||
} else {
|
||
out.push({ key, value: "?" });
|
||
}
|
||
}
|
||
return out;
|
||
}
|
||
|
||
/**
|
||
* Flatten a TSX subtree into element nodes:
|
||
* { tag, attrs, elemChildren, textCount, line, parent }
|
||
*/
|
||
function flattenJsx(node, sf, parent = null, out = []) {
|
||
if (ts.isJsxElement(node)) {
|
||
const el = {
|
||
tag: node.openingElement.tagName.getText(),
|
||
attrs: attrsOf(node.openingElement),
|
||
elemChildren: [],
|
||
textCount: 0,
|
||
line: node.getStart(sf).then ? 0 : sf.getLineAndCharacterOfPosition(node.getStart()).line + 1,
|
||
parent,
|
||
};
|
||
for (const child of node.children) {
|
||
if (ts.isJsxElement(child) || ts.isJsxSelfClosingElement(child)) {
|
||
el.elemChildren.push(flattenJsx(child, sf, el, out));
|
||
} else if (ts.isJsxText(child) && child.getText().trim().length > 0) {
|
||
el.textCount += 1;
|
||
} else if (ts.isJsxExpression(child)) {
|
||
el.textCount += 1; // {expression} counts as dynamic text
|
||
}
|
||
}
|
||
out.push(el);
|
||
return el;
|
||
}
|
||
if (ts.isJsxSelfClosingElement(node)) {
|
||
const el = {
|
||
tag: node.tagName.getText(),
|
||
attrs: attrsOf(node),
|
||
elemChildren: [],
|
||
textCount: 0,
|
||
line: sf.getLineAndCharacterOfPosition(node.getStart()).line + 1,
|
||
parent,
|
||
};
|
||
out.push(el);
|
||
return el;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
for (const p of parsed) {
|
||
p.elements = [];
|
||
const visit = (node) => {
|
||
if (ts.isJsxElement(node) || ts.isJsxSelfClosingElement(node))
|
||
flattenJsx(node, p.sf, null, p.elements);
|
||
ts.forEachChild(node, visit);
|
||
};
|
||
visit(p.sf);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 1. Layout idiom census — exact prop configs for layout primitives
|
||
// ---------------------------------------------------------------------------
|
||
const layoutCounts = new Map(); // sig -> {count, files:Set, examples:[{file,line}]}
|
||
for (const p of parsed) {
|
||
for (const el of p.elements) {
|
||
if (!LAYOUT_TAGS.has(el.tag)) continue;
|
||
const props = el.attrs
|
||
.filter((a) => a.key !== "...")
|
||
.map((a) => `${a.key}=${a.value}`)
|
||
.sort()
|
||
.join(",");
|
||
const sig = `${el.tag}(${props})`;
|
||
const entry = layoutCounts.get(sig) ?? { count: 0, files: new Set(), examples: [] };
|
||
entry.count += 1;
|
||
entry.files.add(p.file);
|
||
if (entry.examples.length < 3) entry.examples.push({ file: p.file, line: el.line });
|
||
layoutCounts.set(sig, entry);
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 2. Structural fingerprints — shape only, text/prop values stripped
|
||
// ---------------------------------------------------------------------------
|
||
const MAX_DEPTH = 4;
|
||
|
||
function shapeSig(el, depth) {
|
||
const attrs = el.attrs
|
||
.map((a) => `${a.key}:${typeof a.value === "string" ? "s" : a.value}`)
|
||
.sort()
|
||
.join(",");
|
||
const kids = depth > 0 ? el.elemChildren.map((c) => shapeSig(c, depth - 1)).join(";") : "";
|
||
const text = el.textCount > 0 ? `#t${el.textCount}` : "";
|
||
return `${el.tag}(${attrs})[${kids}${text}]`;
|
||
}
|
||
|
||
const shapeCounts = new Map(); // sig -> {count, files:Set, examples:[{file,line}]}
|
||
for (const p of parsed) {
|
||
for (const el of p.elements) {
|
||
const sig = shapeSig(el, MAX_DEPTH);
|
||
const entry = shapeCounts.get(sig) ?? { count: 0, files: new Set(), examples: [] };
|
||
entry.count += 1;
|
||
entry.files.add(p.file);
|
||
if (entry.examples.length < 3) entry.examples.push({ file: p.file, line: el.line });
|
||
shapeCounts.set(sig, entry);
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 3. Known idioms
|
||
// ---------------------------------------------------------------------------
|
||
const getAttr = (el, key) => el.attrs.find((a) => a.key === key)?.value;
|
||
|
||
/** Text element with supporting typography + secondary color (the "subtitle"). */
|
||
const isSupporting = (el) =>
|
||
el.tag === "Text" &&
|
||
(getAttr(el, "type") === "supporting" || getAttr(el, "type") === undefined) &&
|
||
(getAttr(el, "color") === "secondary" || getAttr(el, "color") === undefined);
|
||
|
||
function detectIdioms() {
|
||
const results = {
|
||
labelPair: [], // VStack gap=0: [Text(label), Text(supporting/secondary)]
|
||
sectionHeader: [], // VStack gap=0: [Heading, Text(supporting/secondary)]
|
||
statCard: [], // Card padding>=3: VStack gap=1: [supporting, Heading, supporting]
|
||
iconTextRow: [], // HStack gap 1-3 vAlign=center: [Icon, VStack]
|
||
formActions: [], // HStack gap=2: [Button, Button, ...]
|
||
wizardProgress: [], // HStack wrap=wrap: several [Badge, Text] HStacks
|
||
};
|
||
const push = (key, p, el, label) =>
|
||
results[key].push({ file: p.file, line: el.line, detail: label });
|
||
|
||
for (const p of parsed) {
|
||
for (const el of p.elements) {
|
||
const kids = el.elemChildren;
|
||
|
||
// labelPair / sectionHeader
|
||
if (el.tag === "VStack" && getAttr(el, "gap") === "0" && kids.length === 2) {
|
||
const [a, b] = kids;
|
||
if (isSupporting(b) && a.tag === "Text") push("labelPair", p, el, "label+description");
|
||
if (isSupporting(b) && a.tag === "Heading")
|
||
push("sectionHeader", p, el, "heading+description");
|
||
}
|
||
|
||
// statCard
|
||
if (el.tag === "Card" && kids.length === 1) {
|
||
const inner = kids[0];
|
||
const pad = Number(getAttr(el, "padding") ?? "0");
|
||
if (inner.tag === "VStack" && getAttr(inner, "gap") === "1" && pad >= 3) {
|
||
const [t1, h, t2] = inner.elemChildren;
|
||
if (
|
||
t1?.tag === "Text" &&
|
||
isSupporting(t1) &&
|
||
h?.tag === "Heading" &&
|
||
t2?.tag === "Text" &&
|
||
isSupporting(t2)
|
||
) {
|
||
push("statCard", p, el, `Card padding=${pad} + VStack gap=1`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// iconTextRow
|
||
if (el.tag === "HStack" && getAttr(el, "vAlign") === "center") {
|
||
const gap = Number(getAttr(el, "gap") ?? "0");
|
||
if (gap >= 1 && gap <= 3 && kids[0]?.tag === "Icon" && kids[1]?.tag === "VStack") {
|
||
push("iconTextRow", p, el, `Icon + VStack, gap=${gap}`);
|
||
}
|
||
}
|
||
|
||
// formActions
|
||
if (
|
||
el.tag === "HStack" &&
|
||
getAttr(el, "gap") === "2" &&
|
||
kids.length >= 2 &&
|
||
kids.every((k) => k.tag === "Button")
|
||
) {
|
||
push("formActions", p, el, `${kids.length} buttons`);
|
||
}
|
||
|
||
// wizardProgress
|
||
if (el.tag === "HStack" && getAttr(el, "wrap") === "wrap" && kids.length >= 2) {
|
||
const stepy = kids.every(
|
||
(k) =>
|
||
k.tag === "HStack" &&
|
||
k.elemChildren.some((c) => c.tag === "Badge") &&
|
||
k.elemChildren.some((c) => c.tag === "Text"),
|
||
);
|
||
if (stepy) push("wizardProgress", p, el, `${kids.length} steps`);
|
||
}
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
const idioms = detectIdioms();
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 4. File similarity — Jaccard over shape fingerprints (depth 2..3)
|
||
// ---------------------------------------------------------------------------
|
||
function fileSigSet(p) {
|
||
const set = new Set();
|
||
for (const el of p.elements) {
|
||
set.add(shapeSig(el, 3));
|
||
}
|
||
return set;
|
||
}
|
||
const sigSets = new Map(parsed.map((p) => [p.file, fileSigSet(p)]));
|
||
|
||
const pairs = [];
|
||
const fileList = [...sigSets.keys()];
|
||
for (let i = 0; i < fileList.length; i++) {
|
||
const a = sigSets.get(fileList[i]);
|
||
for (let j = i + 1; j < fileList.length; j++) {
|
||
const b = sigSets.get(fileList[j]);
|
||
if (a.size < 15 || b.size < 15) continue;
|
||
let inter = 0;
|
||
for (const s of a) if (b.has(s)) inter += 1;
|
||
const union = a.size + b.size - inter;
|
||
const jac = inter / union;
|
||
if (jac >= 0.3 && inter >= 8) {
|
||
pairs.push({
|
||
a: fileList[i],
|
||
b: fileList[j],
|
||
jaccard: jac,
|
||
shared: inter,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
pairs.sort((x, y) => y.jaccard - x.jaccard || y.shared - x.shared);
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 5. Import cluster census
|
||
// ---------------------------------------------------------------------------
|
||
const importModuleCount = new Map(); // module -> files
|
||
const coImportCount = new Map(); // "mod1|mod2" -> files
|
||
for (const p of parsed) {
|
||
const mods = new Set();
|
||
const visit = (node) => {
|
||
if (
|
||
ts.isImportDeclaration(node) &&
|
||
node.moduleSpecifier &&
|
||
ts.isStringLiteral(node.moduleSpecifier)
|
||
) {
|
||
const mod = node.moduleSpecifier.text;
|
||
if (mod.startsWith("@astryxdesign/core")) mods.add(mod);
|
||
}
|
||
ts.forEachChild(node, visit);
|
||
};
|
||
visit(p.sf);
|
||
const list = [...mods].sort();
|
||
for (const m of list) {
|
||
const e = importModuleCount.get(m) ?? { count: 0, files: new Set() };
|
||
e.count += 1;
|
||
e.files.add(p.file);
|
||
importModuleCount.set(m, e);
|
||
}
|
||
for (let i = 0; i < list.length; i++) {
|
||
for (let j = i + 1; j < list.length; j++) {
|
||
const key = `${list[i]}|${list[j]}`;
|
||
const e = coImportCount.get(key) ?? { count: 0, files: new Set() };
|
||
e.count += 1;
|
||
e.files.add(p.file);
|
||
coImportCount.set(key, e);
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Reporting
|
||
// ---------------------------------------------------------------------------
|
||
const rel = (f) => relative(process.cwd(), f);
|
||
const fmtList = (s) => {
|
||
const arr = [...s].map(rel).sort();
|
||
if (arr.length <= 5) return arr.join(", ");
|
||
return `${arr.slice(0, 5).join(", ")} … (+${arr.length - 5} more)`;
|
||
};
|
||
|
||
const sections = [];
|
||
|
||
// --- layout census ---
|
||
const layoutRows = [...layoutCounts.entries()]
|
||
.filter(([, e]) => e.count >= MIN_COUNT && e.files.size >= 2)
|
||
.sort((a, b) => b[1].count - a[1].count);
|
||
sections.push({
|
||
title: `Layout idiom census — exact VStack/HStack/Card/Grid/Section configs (≥ ${MIN_COUNT} uses, ≥ 2 files)`,
|
||
rows: layoutRows.slice(0, LIMIT).map(([sig, e]) => ({
|
||
signature: sig,
|
||
count: e.count,
|
||
files: e.files.size,
|
||
filesList: fmtList(e.files),
|
||
example: `${rel(e.examples[0].file)}:${e.examples[0].line}`,
|
||
})),
|
||
total: layoutRows.length,
|
||
});
|
||
|
||
// --- shape fingerprints ---
|
||
const shapeRows = [...shapeCounts.entries()]
|
||
.filter(([, e]) => e.count >= MIN_COUNT && e.files.size >= 2)
|
||
.sort((a, b) => b[1].count - a[1].count);
|
||
sections.push({
|
||
title: `Repeated JSX structure fingerprints (≥ ${MIN_COUNT} occurrences, ≥ 2 files)`,
|
||
rows: shapeRows.slice(0, LIMIT).map(([sig, e]) => ({
|
||
signature: sig.length > 220 ? `${sig.slice(0, 220)}…` : sig,
|
||
count: e.count,
|
||
files: e.files.size,
|
||
filesList: fmtList(e.files),
|
||
example: `${rel(e.examples[0].file)}:${e.examples[0].line}`,
|
||
})),
|
||
total: shapeRows.length,
|
||
});
|
||
|
||
// --- idioms ---
|
||
const idiomNames = {
|
||
labelPair: "Label+description pair (VStack gap=0: Text + supporting Text)",
|
||
sectionHeader: "Section header (VStack gap=0: Heading + supporting Text)",
|
||
statCard: "Stat card (Card padding>=3 > VStack gap=1: 2 supporting Texts + Heading)",
|
||
iconTextRow: "Icon + text row (HStack vAlign=center: Icon + VStack)",
|
||
formActions: "Form action row (HStack gap=2: 2+ Buttons)",
|
||
wizardProgress: "Wizard progress (HStack wrap=wrap of Badge+Text steps)",
|
||
};
|
||
const idiomRows = Object.entries(idioms).map(([key, hits]) => {
|
||
const byFile = new Map();
|
||
for (const h of hits) {
|
||
if (!byFile.has(h.file)) byFile.set(h.file, h.line);
|
||
if (byFile.size >= 3) break;
|
||
}
|
||
return {
|
||
idiom: idiomNames[key],
|
||
key,
|
||
count: hits.length,
|
||
files: new Set(hits.map((h) => h.file)).size,
|
||
examples: [...byFile.entries()].map(([f, l]) => `${rel(f)}:${l}`),
|
||
};
|
||
});
|
||
sections.push({
|
||
title: "Known hand-rolled idioms — extraction candidates",
|
||
rows: idiomRows,
|
||
total: idiomRows.length,
|
||
});
|
||
|
||
// --- file similarity ---
|
||
sections.push({
|
||
title: "Most similar page pairs (Jaccard over structure fingerprints)",
|
||
rows: pairs.slice(0, LIMIT).map((p) => ({
|
||
pair: `${rel(p.a)} ↔ ${rel(p.b)}`,
|
||
jaccard: p.jaccard.toFixed(2),
|
||
shared: p.shared,
|
||
})),
|
||
total: pairs.length,
|
||
});
|
||
|
||
// --- import clusters ---
|
||
const coRows = [...coImportCount.entries()]
|
||
.filter(([, e]) => e.count >= 5)
|
||
.sort((a, b) => b[1].count - a[1].count);
|
||
sections.push({
|
||
title: "Most co-imported @astryxdesign/core modules (signals composite components to extract)",
|
||
rows: coRows.slice(0, LIMIT).map(([key, e]) => ({
|
||
modules: key.replace("@astryxdesign/core", ""),
|
||
count: e.count,
|
||
files: e.files.size,
|
||
})),
|
||
total: coRows.length,
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Output
|
||
// ---------------------------------------------------------------------------
|
||
const summary = {
|
||
filesScanned: files.length,
|
||
elementsParsed: parsed.reduce((n, p) => n + p.elements.length, 0),
|
||
vstackUses: [...layoutCounts.entries()]
|
||
.filter(([s]) => s.startsWith("VStack"))
|
||
.reduce((n, [, e]) => n + e.count, 0),
|
||
hstackUses: [...layoutCounts.entries()]
|
||
.filter(([s]) => s.startsWith("HStack"))
|
||
.reduce((n, [, e]) => n + e.count, 0),
|
||
};
|
||
|
||
if (AS_JSON) {
|
||
console.log(JSON.stringify({ summary, sections }, null, 2));
|
||
} else {
|
||
console.log(`\n=== Component duplication scanner ===`);
|
||
console.log(
|
||
`Scanned ${summary.filesScanned} files, ${summary.elementsParsed} JSX elements ` +
|
||
`(${summary.vstackUses} VStack, ${summary.hstackUses} HStack uses).\n`,
|
||
);
|
||
for (const s of sections) {
|
||
console.log(`\n## ${s.title} (${s.total} matches)\n`);
|
||
for (const r of s.rows) {
|
||
if (r.signature !== undefined) {
|
||
console.log(` ${r.count.toString().padStart(4)}× ${r.signature}`);
|
||
console.log(` in ${r.files} files · e.g. ${r.example}`);
|
||
} else if (r.idiom !== undefined) {
|
||
console.log(` ${r.count.toString().padStart(4)}× ${r.idiom}`);
|
||
console.log(` in ${r.files} files · e.g. ${r.examples.join(", ")}`);
|
||
} else if (r.pair !== undefined) {
|
||
console.log(` ${(r.jaccard * 100).toString().padStart(2)}% ${r.pair}`);
|
||
console.log(` ${r.shared} shared structure fingerprints`);
|
||
} else if (r.modules !== undefined) {
|
||
console.log(` ${r.count.toString().padStart(4)}× ${r.modules}`);
|
||
console.log(` in ${r.files} files`);
|
||
}
|
||
}
|
||
}
|
||
console.log("\n");
|
||
}
|