124 lines
3.5 KiB
TypeScript
124 lines
3.5 KiB
TypeScript
import { Application } from "https://deno.land/x/abc@v1.3.3/mod.ts";
|
|
import { logger } from "https://deno.land/x/abc@v1.3.3/middleware/logger.ts";
|
|
import { renderFile } from "https://deno.land/x/dejs@0.10.3/mod.ts";
|
|
import { cleanUrl, scrape } from "./scraper.ts";
|
|
import {
|
|
upsertWebsite,
|
|
getWebsite,
|
|
countWebsites,
|
|
deleteWebsite,
|
|
queryWebsites,
|
|
} from "./data.ts";
|
|
import { embedText } from "./embed.ts";
|
|
|
|
const app = new Application();
|
|
|
|
// app.use(logger());
|
|
app.static("/", "./static");
|
|
|
|
app.renderer = {
|
|
render(name: string, data: any): Promise<Deno.Reader> {
|
|
return renderFile(name, data);
|
|
},
|
|
};
|
|
|
|
app.get("/", async (c) => {
|
|
await c.render("./templates/index.ejs", {});
|
|
});
|
|
|
|
app.get("/search", async (c) => {
|
|
const { query, like, page } = c.queryParams;
|
|
if (!(query || like)) return c.redirect("/");
|
|
if (query && like) throw new Error("you can't have both query and like");
|
|
const start = Date.now();
|
|
const pageNum = parseInt(page) || 0;
|
|
const embedding = query ? await embedText(query) : (await getWebsite(like))?.embedding;
|
|
if (!embedding) throw new Error("failed to get embedding");
|
|
const results = await queryWebsites(embedding, pageNum);
|
|
return c.render("./templates/search.ejs", {
|
|
results,
|
|
query,
|
|
page: pageNum,
|
|
admin: false,
|
|
duration: Date.now() - start,
|
|
});
|
|
});
|
|
|
|
app.get("/search.json", async (c) => {
|
|
const { query, like, page } = c.queryParams;
|
|
if (query && like) throw new Error("you can't have both query and like");
|
|
const pageNum = parseInt(page) || 0;
|
|
const embedding = query ? await embedText(query) : (await getWebsite(like))?.embedding;
|
|
if (!embedding) throw new Error("failed to get embedding");
|
|
const results = await queryWebsites(embedding, pageNum);
|
|
return c.json(results);
|
|
});
|
|
|
|
app.post("/admin/add", async (c) => {
|
|
const { url: rawUrl } = await c.body;
|
|
|
|
if (rawUrl) {
|
|
try {
|
|
const url = cleanUrl(rawUrl);
|
|
const data = await scrape(url);
|
|
const embedding = await embedText(
|
|
`${data.title}\n${data.description}\n${data.content}`,
|
|
);
|
|
if (!embedding) throw new Error("failed to embed text");
|
|
const upserted = await upsertWebsite(data, embedding);
|
|
|
|
console.log(`added url ${data.url}`);
|
|
await c.render("./templates/add.ejs", {
|
|
state: upserted ? "exists" : "success",
|
|
});
|
|
} catch (error) {
|
|
console.error(`failed to add ${rawUrl}: ${error}`);
|
|
console.error(error);
|
|
await c.render("./templates/add.ejs", { state: "error", error }, 400);
|
|
}
|
|
} else {
|
|
await c.render("./templates/add.ejs", { state: "default" });
|
|
}
|
|
});
|
|
|
|
app.get("/admin/add", async (c) => {
|
|
await c.render("./templates/add.ejs", { state: "default" });
|
|
});
|
|
|
|
app.get("/admin/search", async (c) => {
|
|
const { query, page } = c.queryParams;
|
|
if (!query) return c.redirect("/");
|
|
const start = Date.now();
|
|
const pageNum = parseInt(page) || 0;
|
|
const embedding = await embedText(query);
|
|
const results = await queryWebsites(embedding, pageNum);
|
|
await c.render("./templates/search.ejs", {
|
|
results,
|
|
query,
|
|
page: pageNum,
|
|
admin: true,
|
|
duration: Date.now() - start,
|
|
});
|
|
});
|
|
|
|
app.delete("/admin/document/:docid", async (c) => {
|
|
console.log("delete " + c.params.docid);
|
|
deleteWebsite(c.params.docid);
|
|
await c.json({}, 204);
|
|
});
|
|
|
|
app.get("/info", async (c) => {
|
|
await c.render("./templates/info.ejs", {
|
|
indexed: await countWebsites(),
|
|
});
|
|
});
|
|
|
|
app.get("/info.json", async (c) => {
|
|
await c.json({
|
|
indexed: (await countWebsites()).toString(),
|
|
});
|
|
});
|
|
|
|
await app.start({ port: 3974 });
|
|
|
|
console.log("ready");
|