ts: add simple REST endpoint

deno/index.ts

@@ -1,27 +1,11 @@
 import PQueue from "https://deno.land/x/[email protected]/mod.ts"
+import { Application, Router } from "https://deno.land/x/[email protected]/mod.ts";
 
 import * as CSV from './csv.ts';
 import Puppet from './puppet.ts';
 import selectors from './selectors.ts';
 
 const puppet = new Puppet();
-const queue = new PQueue({
-  concurrency: 10,
-  timeout: 60000
-})
-let count = 0
-let statInterval
-queue.addEventListener("active", () =>
-  console.log(`Working on item #${++count}.  Size: ${queue.size}  Pending: ${queue.pending}`))
-queue.addEventListener("next", () =>
-  console.log(`task finished, Size: ${queue.size} Pending: ${queue.pending}`))
-
-queue.addEventListener("idle", async () => {
-  clearInterval(statInterval)
-  await puppet.close()
-  console.log("all done")
-})
-
 async function get_logos(page, selector): {}[] {
   const logos = await page.$$(selector) || [];
   for (const i in logos) {
@@ -36,14 +20,8 @@ async function get_logos(page, selector): {}[] {
   return logos;
 }
 
-
-function process(o: { id: int, url: string, bco: string, name: string }): Promise<void> {
-  const promises: Promise<void>[] = [];
-
-  return puppet.run(async page => {
-    const url = o.url.replace('http:', 'https:');
-    promises.push(new Promise<void>((accept, _reject) => {
-      page.once('load', async () => {
+async function fetch_logos(page, id, dest) {
+  console.error(`getting logos for: ${id}`)
         try {
           const imgs = await get_logos(page, selectors.img_logo);
           const ids = await get_logos(page, selectors.id_logo);
@@ -60,49 +38,68 @@ function process(o: { id: int, url: string, bco: string, name: string }): Promis
               || (bb.height < 10)
               || (bb.x + bb.width < 0)
               || (bb.y + bb.height < 0)) continue;
-            console.log('got bb', o.bco, bb)
+            console.log('got bb',  bb)
 
             try {
-              await logos[i].screenshot({ path: `./data/logos/${o.bco}.logo${i}.png` })
+              await logos[i].screenshot({
+                path: dest
+                  .replace('images', 'logos')
+                  .replace('.png', `.${i}.png`)
+              })
               annotations +=
-                `${o.id} ${bb.x + bb.width / 2} ${bb.y + bb.height / 2} ${bb.width} ${bb.height}\n`
+                `${id} ${bb.x + bb.width / 2} ${bb.y + bb.height / 2} ${bb.width} ${bb.height}\n`
             } catch (e) {
               console.error(`couldn't screenshot logo: ${e}`);
             }
           }
           if (logos.length) {
-            await Deno.writeTextFile(`./data/${o.bco}.chrome.full.txt`, annotations);
-          }
-          await page.screenshot({ path: `./data/${o.bco}.chrome.full.png`, fullPage: true })
-          console.log(`screenshot ok for ${o.name}`);
+            await Deno.writeTextFile(dest
+              .replace('images', 'labels')
+              .replace('png', 'txt'),
+                                     annotations);
+        }
         } catch (err) {
           console.error(`error in screenshot: ${err}`);
         }
-        accept()
-      })
-    }))
+}
+
+const app = new Application();
+const router = new Router();
+
+const stats = {
+  in_flight: 0,
+  done: 0
+}
+router.post('/screenshot', async (ctx) => {
+  const {request, response} = ctx;
+  const q = await request.body().value;
 
-    try {
-      await page.goto(url)
-        .catch(() => page.goto(o.url))
-    } catch (e) {
-      console.error(`got error: ${e}`);
+  stats.in_flight++;
+  const ret = await puppet.run(async page => {
+    console.error('running', q, stats)
+    await page.goto(q.url, {waitUntil: 'networkidle2', timeout: 60000})
+    await page.screenshot({ path: q.path, fullPage: true })
+    if (q.logos) {
+      await fetch_logos(page, q.id, q.logos)
     }
-    await Promise.all(promises);
+    console.error(`screenshot ok: ${q.path}`)
+    return {response: 'ok'}
   })
-}
+  stats.in_flight--;
+  stats.done++
+  response.body = ret
+})
+router.post('/bco', async (ctx) => {
+  const {request, response} = ctx;
+  const q = await request.body().value;
+  const ret = await process(q)
 
-async function run() {
-  let text;
-  try {
-    text = await Deno.readTextFile("./data/entities.csv")
-  } catch (e) {
-    console.error(`couldn't read csv: ${e}`)
-  }
-  if (!text) return setTimeout(run, 1000)
-  statInterval = setInterval(() =>
-    console.log(`Size: ${queue.size}  Pending: ${queue.pending}`), 1000);
+  console.error(`ret: ${ret}`)
+  response.body = ret
+});
 
-  CSV.parse(text, o => queue.add(() => process(o)))
-}
-run()
+app.use(router.routes())
+app.use(router.allowedMethods())
+const addr = '0.0.0.0:8000'
+console.error(`listen on ${addr}`)
+app.listen(addr)