tezlm/celery-search
Archived
1
0
Fork 0
Code Issues Pull requests Releases Activity

get websites that are similar to other websites

Browse source
This commit is contained in:
tezlm 2023-09-09 18:13:58 -07:00
parent 61f9a928f8
commit 6e83e6b203
Signed by: tezlm
GPG key ID: 649733FCD94AFBBA
3 changed files with 19 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1 +0,0 @@
.env

6
data.ts
View file

@ -52,6 +52,12 @@ export async function upsertWebsite(website: WebsiteData, embedding: Array<numbe
}
}
export async function getWebsite(url: string) {
const query = await pg.queryObject`SELECT * FROM websites WHERE url = ${url}`;
const site = query.rows[0];
return site ? { ...site, embedding: JSON.parse(site.embedding) } : null;
}
export async function queryWebsites(embedding: Array<number>, page = 0) {
const limit = 20;
const embedstr = JSON.stringify(embedding);

23
index.ts
View file

@ -4,6 +4,7 @@ import { renderFile } from "https://deno.land/x/dejs@0.10.3/mod.ts";
import { cleanUrl, scrape } from "./scraper.ts";
import {
upsertWebsite,
getWebsite,
countWebsites,
deleteWebsite,
queryWebsites,
@ -26,14 +27,15 @@ app.get("/", async (c) => {
});
app.get("/search", async (c) => {
const { query, page } = c.queryParams;
if (!query) return c.redirect("/");
const { query, like, page } = c.queryParams;
if (!(query || like)) return c.redirect("/");
if (query && like) throw new Error("you can't have both query and like");
const start = Date.now();
const pageNum = parseInt(page) || 0;
const embedding = await embedText(query);
if (!embedding) throw new Error("failed to embed query");
const pageNum = parseInt(page) || 0;
const embedding = query ? await embedText(query) : (await getWebsite(like))?.embedding;
if (!embedding) throw new Error("failed to get embedding");
const results = await queryWebsites(embedding, pageNum);
await c.render("./templates/search.ejs", {
return c.render("./templates/search.ejs", {
results,
query,
page: pageNum,
@ -43,12 +45,13 @@ app.get("/search", async (c) => {
});
app.get("/search.json", async (c) => {
const { query, page } = c.queryParams;
const { query, like, page } = c.queryParams;
if (query && like) throw new Error("you can't have both query and like");
const pageNum = parseInt(page) || 0;
const embedding = await embedText(query);
if (!embedding) throw new Error("failed to embed query");
const embedding = query ? await embedText(query) : (await getWebsite(like))?.embedding;
if (!embedding) throw new Error("failed to get embedding");
const results = await queryWebsites(embedding, pageNum);
await c.json(results);
return c.json(results);
});
app.post("/admin/add", async (c) => {