debugging the api..

.gitignore

@@ -3,7 +3,7 @@ node_modules
 *.bin
 .DS_Store
 .venv
-models/
-sandbox/
-audio.pipe
-video.pipe
+./database/completed/*.json
+./database/pending/*.json
+*.mp4
+sandbox

src/database/constants.mts

@@ -1,3 +1,6 @@
 
 export const pendingTasksDirFilePath = './database/pending/'
 export const completedTasksDirFilePath = './database/completed/'
+
+export const shotFormatVersion = 1
+export const sequenceFormatVersion = 1

src/index.mts

@@ -3,7 +3,7 @@ import { createReadStream, promises as fs } from "fs"
 import express from "express"
 
 import { VideoTask, VideoSequenceRequest } from "./types.mts"
-import { requestToTask } from "./services/requestToTask.mts"
+import { parseVideoRequest } from "./utils/parseVideoRequest.mts"
 import { savePendingTask } from "./database/savePendingTask.mts"
 import { getTask } from "./database/getTask.mts"
 import { main } from "./main.mts"
@@ -31,7 +31,7 @@ app.post("/", async (req, res) => {
 
   console.log(`creating task from request..`)
   try {
-    task = await requestToTask(request)
+    task = await parseVideoRequest(request)
   } catch (err) {
     console.error(`failed to create task: ${task}`)
     res.status(400)
@@ -79,18 +79,9 @@ app.get("/video/:id\.mp4", async (req, res) => {
   }
 
   let task: VideoTask = null
-
   try {
     task = await getTask(req.params.id)
     console.log("returning result to user..")
-
-    const filePath = task.finalFilePath || task.tmpFilePath || ''
-    if (!filePath) {
-      res.status(400)
-      res.write(JSON.stringify({ error: "video exists, but cannot be previewed yet" }))
-      res.end()
-      return
-    }
   } catch (err) {
     res.status(404)
     res.write(JSON.stringify({ error: "this video doesn't exist" }))
@@ -98,6 +89,15 @@ app.get("/video/:id\.mp4", async (req, res) => {
     return
   }
 
+
+  const filePath = task.finalFilePath || task.tmpFilePath || ''
+  if (!filePath) {
+    res.status(400)
+    res.write(JSON.stringify({ error: "video exists, but cannot be previewed yet" }))
+    res.end()
+    return
+  }
+
   // file path exists, let's try to read it
   try {
     // do we need this?

src/types.mts

@@ -124,8 +124,6 @@ export interface VideoShotMeta {
   actorDialoguePrompt: string
 
   seed: number
-  upscale: boolean
-
   noise: boolean // add movie noise
 
   durationMs: number // in milliseconds
@@ -133,7 +131,7 @@ export interface VideoShotMeta {
 
   fps: number // 8, 12, 24, 30, 60
 
-  resolution: number // 256, 512, 576, 720, 1080
+  resolution: string // {width}x{height} (256, 512, 576, 720, 1080)
 
   introTransition: VideoTransition
   introDurationMs: number // in milliseconds
@@ -151,6 +149,10 @@ export interface VideoShotMeta {
 
 
 export interface VideoShotData {
+
+  // used to check compatibility
+  version: number
+
   hasGeneratedVideo: boolean
   hasUpscaledVideo: boolean
   hasGeneratedBackgroundAudio: boolean
@@ -194,7 +196,6 @@ export interface VideoSequenceMeta {
   actorDialoguePrompt: string
 
   seed: number
-  upscale: boolean
 
   noise: boolean // add movie noise
 
@@ -210,6 +211,10 @@ export interface VideoSequenceMeta {
 
 
 export interface VideoSequenceData {
+
+  // used to check compatibility
+  version: number
+
   nbCompletedShots: number
   nbTotalShots: number
   progressPercent: number

src/utils/parseShotRequest.mts

@@ -0,0 +1,85 @@
+import { v4 as uuidv4 } from "uuid"
+
+// convert a request (which might be invalid)
+
+import { VideoSequence, VideoShot, VideoShotMeta } from "../types.mts"
+import { generateSeed } from "../services/generateSeed.mts"
+import { getValidNumber } from "./getValidNumber.mts"
+import { shotFormatVersion } from "../database/constants.mts"
+
+export const parseShotRequest = async (sequence: VideoSequence, maybeShotMeta: VideoShotMeta): Promise<VideoShot> => {
+
+  const shot: VideoShot = {
+    id: maybeShotMeta.id || uuidv4(),
+
+    shotPrompt: `${maybeShotMeta.shotPrompt || ""}`,
+
+    // describe the background audio (crowd, birds, wind, sea etc..)
+    backgroundAudioPrompt: `${maybeShotMeta.backgroundAudioPrompt || ""}`,
+
+    // describe the foreground audio (cars revving, footsteps, objects breaking, explosion etc)
+    foregroundAudioPrompt: `${maybeShotMeta.foregroundAudioPrompt || ""}`,
+
+    // describe the main actor visible in the shot (optional)
+    actorPrompt: `${maybeShotMeta.actorPrompt || ""}`,
+
+    // describe the main actor voice (man, woman, old, young, amused, annoyed.. etc)
+    actorVoicePrompt: `${maybeShotMeta.actorVoicePrompt || ""}`,
+
+    // describe the main actor dialogue line
+    actorDialoguePrompt: `${maybeShotMeta.actorDialoguePrompt || ""}`,
+
+    // a video sequence SHOULD NOT HAVE a consistent seed, to avoid weird geometry similarities
+    seed: getValidNumber(maybeShotMeta.seed, 0, 4294967295, generateSeed()),
+
+    // a video sequence SHOULD HAVE a consistent grain
+    noise: sequence.noise,
+
+    // a video sequence CAN HAVE inconsistent scene duration, like in any real movie
+    durationMs: getValidNumber(maybeShotMeta.durationMs, 0, 6000, 3000),
+    
+    // a video sequence CAN HAVE inconsistent iteration steps
+    steps: getValidNumber(maybeShotMeta.steps || sequence.steps, 1, 60, 35),
+
+    // a video sequence MUST HAVE consistent frames per second
+    fps: getValidNumber(sequence.fps, 8, 60, 24),
+
+    // a video sequence MUST HAVE a consistent resolution
+    resolution: sequence.resolution,
+
+    // a video sequence CAN HAVE intro transitions for each shot
+    introTransition: 'fade',
+    introDurationMs: 500,
+
+    // for internal use
+
+    version: shotFormatVersion,
+    hasGeneratedVideo: false,
+    hasUpscaledVideo: false,
+    hasGeneratedBackgroundAudio: false,
+    hasGeneratedForegroundAudio: false,
+    hasGeneratedActor: false,
+    hasInterpolatedVideo: false,
+    hasAddedAudio: false,
+    hasPostProcessedVideo: false,
+    nbCompletedSteps: 0,
+
+    // 0. in queue
+    // 1. generate with Zeroscope
+    // 2. upscale with Zeroscope XL
+    // 3. interpolate with FILE
+    // 4. generate audio background
+    // 5. generate audio foreground
+    // 6. add audio to video
+    // 7. post-processing
+    nbTotalSteps: 7,
+    progressPercent: 0,
+    completedAt: '',
+    completed: false,
+    error: '',
+    tmpFilePath: '',
+    finalFilePath: '',
+  }
+
+  return shot
+}

src/{services/requestToTask.mts → utils/parseVideoRequest.mts}

@@ -3,12 +3,14 @@ import { v4 as uuidv4 } from "uuid"
 // convert a request (which might be invalid)
 
 import { VideoSequenceRequest, VideoTask } from "../types.mts"
-import { generateSeed } from "./generateSeed.mts"
-import { getValidNumber } from "../utils/getValidNumber.mts"
-import { getValidResolution } from "../utils/getValidResolution.mts"
+import { generateSeed } from "../services/generateSeed.mts"
+import { getValidNumber } from "./getValidNumber.mts"
+import { getValidResolution } from "./getValidResolution.mts"
+import { parseShotRequest } from "./parseShotRequest.mts"
+import { sequenceFormatVersion } from "../database/constants.mts"
 
-// into a valid task
-export const requestToTask = async (request: VideoSequenceRequest): Promise<VideoTask> => {
+
+export const parseVideoRequest = async (request: VideoSequenceRequest): Promise<VideoTask> => {
 
   const task: VideoTask = {
     // ------------ VideoSequenceMeta -------------
@@ -34,8 +36,6 @@ export const requestToTask = async (request: VideoSequenceRequest): Promise<Vide
 
     seed: getValidNumber(request.sequence.seed, 0, 4294967295, generateSeed()),
 
-    upscale: request.sequence.upscale === true,
-
     noise: request.sequence.noise === true,
 
     steps: getValidNumber(request.sequence.steps, 1, 60, 35),
@@ -48,6 +48,7 @@ export const requestToTask = async (request: VideoSequenceRequest): Promise<Vide
     outroDurationMs: 3000,
 
     // ---------- VideoSequenceData ---------
+    version: sequenceFormatVersion,
     nbCompletedShots: 0,
     nbTotalShots: 0,
     progressPercent: 0,
@@ -62,54 +63,17 @@ export const requestToTask = async (request: VideoSequenceRequest): Promise<Vide
     shots: [],
   }
 
+  const maybeShots = Array.isArray(request.shots) ? request.shots : []
 
-  // optional background audio prompt
-  const backgroundAudioPrompt = `${query.backgroundAudioPrompt || ""}`
-
-  // optional foreground audio prompt
-  const foregroundAudioPrompt = `${query.foregroundAudioPrompt || ""}`
-
-  // optional seed
-  const defaultSeed = generateSeed()
-  const seedStr = getValidNumber(Number(`${query.seed || defaultSeed}`)
-  const maybeSeed = Number(seedStr)
-  const seed = isNaN(maybeSeed) || ! isFinite(maybeSeed) ? defaultSeed : maybeSeed
-  
-  // in production we want those ON by default
-  const upscale = `${query.upscale || "true"}` === "true"
-  const interpolate = `${query.upscale || "true"}` === "true"
-  const noise = `${query.noise || "true"}` === "true"
-
-
-  const defaultDuration = 3
-  const maxDuration = 5
-  const durationStr = Number(`${query.duration || defaultDuration}`)
-  const maybeDuration = Number(durationStr)
-  const duration = Math.min(maxDuration, Math.max(1, isNaN(maybeDuration) || !isFinite(maybeDuration) ? defaultDuration : maybeDuration))
-  
-  const defaultSteps = 35
-  const stepsStr = Number(`${query.steps || defaultSteps}`)
-  const maybeSteps = Number(stepsStr)
-  const nbSteps = Math.min(60, Math.max(1, isNaN(maybeSteps) || !isFinite(maybeSteps) ? defaultSteps : maybeSteps))
-  
-  // const frames per second
-  const defaultFps = 24
-  const fpsStr = Number(`${query.fps || defaultFps}`)
-  const maybeFps = Number(fpsStr)
-  const nbFrames = Math.min(60, Math.max(8, isNaN(maybeFps) || !isFinite(maybeFps) ? defaultFps : maybeFps))
-  
-  const defaultResolution = 576
-  const resolutionStr = Number(`${query.resolution || defaultResolution}`)
-  const maybeResolution = Number(resolutionStr)
-  const resolution = Math.min(1080, Math.max(256, isNaN(maybeResolution) || !isFinite(maybeResolution) ? defaultResolution : maybeResolution))
-  
-  const actorPrompt = `${query.actorPrompt || ""}`
-
-  const actorVoicePrompt = `${query.actorVoicePrompt || ""}`
-
-  const actorDialoguePrompt = `${query.actorDialoguePrompt || ""}`
-
+  for (const maybeShot of maybeShots) {
+    try {
+      const shot = await parseShotRequest(task, maybeShot)
+      task.shots.push(shot)
+    } catch (err) {
+      console.log(`error parsing shot: `, maybeShot)
+    }
 
+  }
 
   return task
 }