celery-search/index.ts

import { Application } from "https://deno.land/x/abc@v1.3.3/mod.ts";
import { logger } from "https://deno.land/x/abc@v1.3.3/middleware/logger.ts";
import { renderFile } from "https://deno.land/x/dejs@0.10.3/mod.ts";
import { cleanUrl, scrape } from "./scraper.ts";
import {
  upsertWebsite,
  getWebsite,
  countWebsites,
  deleteWebsite,
  queryWebsites,
} from "./data.ts";
import { embedText } from "./embed.ts";

const app = new Application();

// app.use(logger());
app.static("/", "./static");

app.renderer = {
  render(name: string, data: any): Promise<Deno.Reader> {
    return renderFile(name, data);
  },
};

app.get("/", async (c) => {
  await c.render("./templates/index.ejs", {});
});

app.get("/search", async (c) => {
  const { query, like, page } = c.queryParams;
  if (!(query || like)) return c.redirect("/");
  if (query && like) throw new Error("you can't have both query and like");
  const start = Date.now();
  const pageNum = parseInt(page) || 0;  
  const embedding = query ? await embedText(query) : (await getWebsite(like))?.embedding;
  if (!embedding) throw new Error("failed to get embedding");
  const results = await queryWebsites(embedding, pageNum);
  return c.render("./templates/search.ejs", {
    results,
    query,
    page: pageNum,
    admin: false,
    duration: Date.now() - start,
  });
});

app.get("/search.json", async (c) => {
  const { query, like, page } = c.queryParams;
  if (query && like) throw new Error("you can't have both query and like");
  const pageNum = parseInt(page) || 0;
  const embedding = query ? await embedText(query) : (await getWebsite(like))?.embedding;
  if (!embedding) throw new Error("failed to get embedding");
  const results = await queryWebsites(embedding, pageNum);
  return c.json(results);
});

app.post("/admin/add", async (c) => {
  const { url: rawUrl } = await c.body;

  if (rawUrl) {
    try {
      const url = cleanUrl(rawUrl);
      const data = await scrape(url);
      const embedding = await embedText(
        `${data.title}\n${data.description}\n${data.content}`,
      );
      if (!embedding) throw new Error("failed to embed text");
      const upserted = await upsertWebsite(data, embedding);

      console.log(`added url ${data.url}`);
      await c.render("./templates/add.ejs", {
        state: upserted ? "exists" : "success",
      });
    } catch (error) {
      console.error(`failed to add ${rawUrl}: ${error}`);
      console.error(error);
      await c.render("./templates/add.ejs", { state: "error", error }, 400);
    }
  } else {
    await c.render("./templates/add.ejs", { state: "default" });
  }
});

app.get("/admin/add", async (c) => {
  await c.render("./templates/add.ejs", { state: "default" });
});

app.get("/admin/search", async (c) => {
  const { query, page } = c.queryParams;
  if (!query) return c.redirect("/");
  const start = Date.now();
  const pageNum = parseInt(page) || 0;
  const embedding = await embedText(query);
  const results = await queryWebsites(embedding, pageNum);
  await c.render("./templates/search.ejs", {
    results,
    query,
    page: pageNum,
    admin: true,
    duration: Date.now() - start,
  });
});

app.delete("/admin/document/:docid", async (c) => {
  console.log("delete " + c.params.docid);
  deleteWebsite(c.params.docid);
  await c.json({}, 204);
});

app.get("/info", async (c) => {
  await c.render("./templates/info.ejs", {
    indexed: await countWebsites(),
  });
});

app.get("/info.json", async (c) => {
  await c.json({
    indexed: (await countWebsites()).toString(),
  });
});

await app.start({ port: 3974 });

console.log("ready");