참고소스 수정본

참고/firecrawl-main/apps/api/utils/find_uncovered_files.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Get all files tracked by git
+git ls-files > /tmp/all_files.txt
+
+# Get files matched by CODEOWNERS
+while read -r line; do
+  # Skip comments and empty lines
+  [[ "$line" =~ ^#.*$ ]] && continue
+  [[ -z "$line" ]] && continue
+  
+  # Extract the path pattern
+  pattern=$(echo "$line" | awk '{print $1}')
+  
+  # Convert the pattern to a form git understands
+  # Remove leading slash if present
+  pattern=${pattern#/}
+  
+  # List files matching this pattern
+  git ls-files "$pattern" 2>/dev/null >> /tmp/covered_files.txt
+done < .github/CODEOWNERS
+
+# Sort and get unique entries
+sort -u /tmp/covered_files.txt > /tmp/covered_files_unique.txt
+
+# Find files that are in all_files but not in covered_files
+comm -23 /tmp/all_files.txt /tmp/covered_files_unique.txt
+
+# Cleanup
+rm /tmp/all_files.txt /tmp/covered_files.txt /tmp/covered_files_unique.txt

참고/firecrawl-main/apps/api/utils/logview.js

@@ -0,0 +1,38 @@
+const fs = require("fs");
+
+// METHOD: Winston log file
+// const logs = fs.readFileSync("7a373219-0eb4-4e47-b2df-e90e12afd5c1.log", "utf8")
+//     .split("\n").filter(x => x.trim().length > 0).map(x => JSON.parse(x));
+
+// METHOD: GCloud export
+const logs = [
+    "downloaded-logs-20241213-225607.json",
+    "downloaded-logs-20241213-225654.json",
+    "downloaded-logs-20241213-225720.json",
+    "downloaded-logs-20241213-225758.json",
+    "downloaded-logs-20241213-225825.json",
+    "downloaded-logs-20241213-225843.json",
+].flatMap(x => JSON.parse(fs.readFileSync(x, "utf8"))).map(x => x.jsonPayload);
+
+
+const crawlIds = [...new Set(logs.map(x => x.crawlId).filter(x => x))];
+
+const urlFilter = x => new URL(x).pathname.slice(1) || "root"
+
+for (const crawlId of crawlIds) {
+    const crawlLogs = logs.filter(x => x.crawlId === crawlId);
+    fs.writeFileSync("crawl-" + crawlId + ".log", crawlLogs.map(x => JSON.stringify(x)).join("\n"));
+
+    const jobAdds = crawlLogs.filter(x => x.jobPriority !== undefined && x.message.startsWith("Added job for URL "));
+    const jobStarts = crawlLogs.filter(x => x.message.startsWith("🐂 Worker taking job"));
+    const ttl = [...new Set(crawlLogs.filter(x => x.method === "lockURL" && x.res !== undefined).map(x => x.url))]
+
+    fs.writeFileSync(crawlId + ".md",
+        "```mermaid\nflowchart LR\n    "
+            + jobStarts.map(x => `${x.jobId}[${urlFilter(x.url)}]`).join("\n    ") + "\n    "
+            + jobAdds.map(x => `${x.jobId}[${urlFilter(jobStarts.find(y => y.jobId === x.jobId).url)}] --> ${x.newJobId}[${urlFilter(x.url)}]`).join("\n    ")
+            + "\n```\n\nURLs scraped: (" + jobStarts.length + ")\n"
+            + jobStarts.map(x => "- " + x.url).join("\n") + "\n\nURLs tried to lock: (" + ttl.length + ")\n"
+            + ttl.map(x => "- " + x + " ("+ crawlLogs.filter(y => y.method === "lockURL" && y.res !== undefined && y.url === x).length + "; " + crawlLogs.filter(y => y.method === "lockURL" && y.res === true && y.url === x).length + ")").join("\n")
+    );
+}

참고/firecrawl-main/apps/api/utils/urldump-redis.js

@@ -0,0 +1,14 @@
+require("dotenv").config();
+const Redis = require("ioredis");
+
+const crawlId = process.argv[2];
+
+const redisConnection = new Redis(process.env.REDIS_URL, {
+    maxRetriesPerRequest: null,
+});
+
+(async () => {
+    const res = await redisConnection.sscan("crawl:" + crawlId + ":visited_unique", 0, "COUNT", 999);
+    await require("fs/promises").writeFile(crawlId + "-visited.txt", res[1].map(x => x.split("://").slice(1).join("://")).sort().join("\n"));
+    process.exit(0);
+})();

참고/firecrawl-main/apps/api/utils/urldump.js

@@ -0,0 +1,43 @@
+require("dotenv").config();
+
+//const baseUrl = "https://api.firecrawl.dev";
+const baseUrl = "http://localhost:3002";
+const crawlId = process.argv[2];
+
+(async () => {
+    let url = baseUrl + "/v1/crawl/" + crawlId;
+    let urls = [];
+
+    while (url) {
+        let res;
+        
+        while (true) {
+            try {
+                res = (await (await fetch(url, {
+                    headers: {
+                        "Authorization": "Bearer " + process.env.TEST_API_KEY
+                    }
+                })).json());
+                break;
+            } catch (e) {
+                console.error(e);
+            }
+        }
+
+        console.log(res.data.length);
+        if (res.data.length === 0) {
+            break;
+        }
+
+        urls.push(...res.data.map(x => x.metadata.url ?? x.metadata.sourceURL));
+
+        url = res.next;
+        if (url !== undefined) {
+            const o = new URL(url)
+            o.protocol = new URL(baseUrl).protocol;
+            url = o.href;
+        }
+    }
+
+    await require("fs/promises").writeFile(crawlId + "-urls.txt", urls.map(x => x.split("://").slice(1).join("://")).sort().join("\n"));
+})();