Video support, comment section, more
This commit is contained in:
+263
-12
@@ -119,6 +119,17 @@ const env = {
|
||||
gl.texImage2D(target, level, internalFormat, w, h, border, format, type, data);
|
||||
},
|
||||
glTexParameteri: (target, pname, param) => gl.texParameteri(target, pname, param),
|
||||
glTexSubImage2D(target, level, x, y, w, h, format, type, pixels) {
|
||||
gl.texSubImage2D(target, level, x, y, w, h, format, type, u8().subarray(Number(pixels), Number(pixels) + w * h * 4));
|
||||
},
|
||||
glDeleteTextures(n, ptr) {
|
||||
for (let i = 0; i < n; i++) {
|
||||
const h = dv().getUint32(Number(ptr) + i * 4, true);
|
||||
if (objects[h]) gl.deleteTexture(objects[h]);
|
||||
objects[h] = null;
|
||||
}
|
||||
},
|
||||
glGenerateMipmap: (target) => gl.generateMipmap(target),
|
||||
glPixelStorei: (pname, param) => gl.pixelStorei(pname, param),
|
||||
glEnable: (cap) => gl.enable(cap),
|
||||
glDisable: (cap) => gl.disable(cap),
|
||||
@@ -158,21 +169,245 @@ const env = {
|
||||
setTimeout(() => URL.revokeObjectURL(a.href), 1000);
|
||||
},
|
||||
js_request_import() { importInput.click(); },
|
||||
js_http_post(tag, path, pathCount, body, bodyCount, token, tokenCount) {
|
||||
const headers = { 'Content-Type': 'text/plain; charset=utf-8' };
|
||||
js_http(tag, method, methodCount, path, pathCount, body, bodyCount, token, tokenCount, key, keyCount) {
|
||||
const verb = str(method, methodCount);
|
||||
const headers = {};
|
||||
if (verb !== 'GET') headers['Content-Type'] = 'text/plain; charset=utf-8';
|
||||
if (tokenCount) headers['Authorization'] = 'Bearer ' + str(token, tokenCount);
|
||||
const request = fetch(str(path, pathCount), { method: 'POST', headers, body: u8().slice(Number(body), Number(body) + Number(bodyCount)) });
|
||||
request.then(async (response) => [response.status, new Uint8Array(await response.arrayBuffer())])
|
||||
.catch(() => [0, new Uint8Array(0)])
|
||||
.then(([status, bytes]) => {
|
||||
const ptr = bytes.length ? X.web_alloc(BigInt(bytes.length)) : 0n;
|
||||
if (bytes.length) u8().set(bytes, Number(ptr));
|
||||
X.web_http_response(tag, status, ptr, BigInt(bytes.length));
|
||||
requestFrame();
|
||||
});
|
||||
const keyText = str(key, keyCount);
|
||||
const options = { method: verb, headers };
|
||||
if (verb !== 'GET') options.body = u8().slice(Number(body), Number(body) + Number(bodyCount));
|
||||
fetch(str(path, pathCount), options)
|
||||
.then(async (response) => [response.status, new Uint8Array(await response.arrayBuffer())])
|
||||
.catch(() => [0, new Uint8Array(0)])
|
||||
.then(([status, bytes]) => {
|
||||
const [kp, kn] = toWasm(encoder.encode(keyText));
|
||||
const [bp, bn] = toWasm(bytes);
|
||||
X.web_http_response(tag, status, bp, bn, kp, kn);
|
||||
requestFrame();
|
||||
});
|
||||
},
|
||||
|
||||
// Images are decoded by the browser, scaled down to at most 2048 px, and handed back as RGBA.
|
||||
js_decode_image(key, keyCount, data, count) {
|
||||
const keyText = str(key, keyCount);
|
||||
const blob = new Blob([u8().slice(Number(data), Number(data) + Number(count))]);
|
||||
decodeImage(blob).then((image) => {
|
||||
const [kp, kn] = toWasm(encoder.encode(keyText));
|
||||
if (!image) { X.web_image_decoded(kp, kn, 0, 0, 0, 0n); requestFrame(); return; }
|
||||
const [pp] = toWasm(image.pixels);
|
||||
X.web_image_decoded(kp, kn, 1, image.width, image.height, pp);
|
||||
requestFrame();
|
||||
});
|
||||
},
|
||||
|
||||
// Size, duration and a PNG poster (first frame) of a video.
|
||||
js_probe_video(key, keyCount, data, count) {
|
||||
const keyText = str(key, keyCount);
|
||||
const url = URL.createObjectURL(new Blob([u8().slice(Number(data), Number(data) + Number(count))]));
|
||||
probeVideo(url).then((info) => {
|
||||
URL.revokeObjectURL(url);
|
||||
const [kp, kn] = toWasm(encoder.encode(keyText));
|
||||
if (!info) { X.web_video_probed(kp, kn, 0, 0, 0, 0, 0n, 0n); requestFrame(); return; }
|
||||
const [pp, pn] = toWasm(info.png);
|
||||
X.web_video_probed(kp, kn, 1, info.width, info.height, info.duration, pp, pn);
|
||||
requestFrame();
|
||||
});
|
||||
},
|
||||
|
||||
// Local projects keep attachments in IndexedDB (localStorage is far too small).
|
||||
js_blob_save(key, keyCount, data, count) {
|
||||
const keyText = str(key, keyCount);
|
||||
const bytes = u8().slice(Number(data), Number(data) + Number(count));
|
||||
idb('readwrite', (store) => store.put(bytes, keyText));
|
||||
},
|
||||
js_blob_load(key, keyCount) {
|
||||
const keyText = str(key, keyCount);
|
||||
idb('readonly', (store) => store.get(keyText)).then((bytes) => {
|
||||
const [kp, kn] = toWasm(encoder.encode(keyText));
|
||||
if (!bytes) { X.web_blob_loaded(kp, kn, 0, 0n, 0n); requestFrame(); return; }
|
||||
const [dp, dn] = toWasm(bytes);
|
||||
X.web_blob_loaded(kp, kn, 1, dp, dn);
|
||||
requestFrame();
|
||||
}).catch(() => {
|
||||
const [kp, kn] = toWasm(encoder.encode(keyText));
|
||||
X.web_blob_loaded(kp, kn, 0, 0n, 0n);
|
||||
requestFrame();
|
||||
});
|
||||
},
|
||||
|
||||
js_pick_files() { mediaInput.click(); },
|
||||
|
||||
// Grabs one frame of a screen, window or tab the user picks.
|
||||
js_capture_screenshot() {
|
||||
captureScreen().then((blob) => { if (blob) addFile(screenshotName(), blob); })
|
||||
.catch((e) => console.warn('Screen capture cancelled:', e));
|
||||
},
|
||||
|
||||
// One video at a time plays in an off-screen <video>; its frames are uploaded into the given texture.
|
||||
js_video_open(data, count) {
|
||||
closeVideo();
|
||||
const url = URL.createObjectURL(new Blob([u8().slice(Number(data), Number(data) + Number(count))]));
|
||||
const v = document.createElement('video');
|
||||
v.src = url;
|
||||
v.playsInline = true;
|
||||
v.muted = videoMuted;
|
||||
v.preload = 'auto';
|
||||
player = { video: v, url, failed: false };
|
||||
v.addEventListener('error', () => { if (player && player.video === v) player.failed = true; requestFrame(); });
|
||||
for (const name of ['loadeddata', 'play', 'pause', 'seeked', 'ended']) v.addEventListener(name, requestFrame);
|
||||
v.play().catch(() => {});
|
||||
return 1;
|
||||
},
|
||||
js_video_update(texture, state) {
|
||||
const d = dv();
|
||||
const base = Number(state);
|
||||
if (!player) return;
|
||||
const v = player.video;
|
||||
const ready = v.readyState >= 2 && v.videoWidth > 0;
|
||||
if (ready) {
|
||||
gl.bindTexture(gl.TEXTURE_2D, objects[texture]);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, v);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
||||
}
|
||||
d.setFloat64(base + 0, v.currentTime || 0, true);
|
||||
d.setFloat64(base + 8, isFinite(v.duration) ? v.duration : 0, true);
|
||||
d.setInt32(base + 16, (!v.paused && !v.ended) ? 1 : 0, true);
|
||||
d.setInt32(base + 20, ready ? 1 : 0, true);
|
||||
d.setInt32(base + 24, v.videoWidth, true);
|
||||
d.setInt32(base + 28, v.videoHeight, true);
|
||||
d.setInt32(base + 32, player.failed ? 1 : 0, true);
|
||||
},
|
||||
js_video_set_playing(playing) {
|
||||
if (!player) return;
|
||||
if (playing) player.video.play().catch(() => {});
|
||||
else player.video.pause();
|
||||
},
|
||||
js_video_seek(seconds) { if (player) player.video.currentTime = seconds; },
|
||||
js_video_set_muted(muted) { videoMuted = !!muted; if (player) player.video.muted = videoMuted; },
|
||||
js_video_close() { closeVideo(); },
|
||||
};
|
||||
|
||||
//
|
||||
// Media helpers.
|
||||
//
|
||||
|
||||
let player = null;
|
||||
let videoMuted = false;
|
||||
|
||||
function closeVideo() {
|
||||
if (!player) return;
|
||||
player.video.pause();
|
||||
player.video.removeAttribute('src');
|
||||
player.video.load();
|
||||
URL.revokeObjectURL(player.url);
|
||||
player = null;
|
||||
}
|
||||
|
||||
// Copies bytes into WASM memory (allocated with web_alloc; Jai frees them). Returns [pointer, length] as BigInts.
|
||||
function toWasm(bytes) {
|
||||
if (!bytes || !bytes.length) return [0n, 0n];
|
||||
const ptr = X.web_alloc(BigInt(bytes.length));
|
||||
u8().set(bytes, Number(ptr));
|
||||
return [ptr, BigInt(bytes.length)];
|
||||
}
|
||||
|
||||
const MAX_IMAGE_SIDE = 2048;
|
||||
|
||||
async function decodeImage(blob) {
|
||||
try {
|
||||
let bitmap = await createImageBitmap(blob);
|
||||
const scale = Math.min(1, MAX_IMAGE_SIDE / Math.max(bitmap.width, bitmap.height));
|
||||
const w = Math.max(1, Math.round(bitmap.width * scale)), h = Math.max(1, Math.round(bitmap.height * scale));
|
||||
const canvas2 = new OffscreenCanvas(w, h);
|
||||
const ctx = canvas2.getContext('2d');
|
||||
ctx.drawImage(bitmap, 0, 0, w, h);
|
||||
bitmap.close();
|
||||
const data = ctx.getImageData(0, 0, w, h).data;
|
||||
return { width: w, height: h, pixels: new Uint8Array(data.buffer, data.byteOffset, data.byteLength) };
|
||||
} catch (e) {
|
||||
console.warn('Could not decode image:', e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function probeVideo(url) {
|
||||
return new Promise((resolve) => {
|
||||
const v = document.createElement('video');
|
||||
v.muted = true;
|
||||
v.playsInline = true;
|
||||
v.preload = 'auto';
|
||||
const timer = setTimeout(() => resolve(null), 20000);
|
||||
v.addEventListener('error', () => { clearTimeout(timer); resolve(null); });
|
||||
v.addEventListener('loadeddata', () => { v.currentTime = Math.min(0.1, (v.duration || 1) / 2); });
|
||||
v.addEventListener('seeked', async () => {
|
||||
clearTimeout(timer);
|
||||
const scale = Math.min(1, 1280 / Math.max(v.videoWidth, v.videoHeight));
|
||||
const c = new OffscreenCanvas(Math.max(1, Math.round(v.videoWidth * scale)), Math.max(1, Math.round(v.videoHeight * scale)));
|
||||
c.getContext('2d').drawImage(v, 0, 0, c.width, c.height);
|
||||
const png = new Uint8Array(await (await c.convertToBlob({ type: 'image/png' })).arrayBuffer());
|
||||
resolve({ width: v.videoWidth, height: v.videoHeight, duration: isFinite(v.duration) ? v.duration : 0, png });
|
||||
}, { once: true });
|
||||
v.src = url;
|
||||
});
|
||||
}
|
||||
|
||||
async function captureScreen() {
|
||||
const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false });
|
||||
try {
|
||||
const v = document.createElement('video');
|
||||
v.srcObject = stream;
|
||||
v.muted = true;
|
||||
await v.play();
|
||||
await new Promise((r) => setTimeout(r, 300)); // Let the first real frame arrive.
|
||||
const c = new OffscreenCanvas(v.videoWidth, v.videoHeight);
|
||||
c.getContext('2d').drawImage(v, 0, 0);
|
||||
return await c.convertToBlob({ type: 'image/png' });
|
||||
} finally {
|
||||
for (const track of stream.getTracks()) track.stop();
|
||||
}
|
||||
}
|
||||
|
||||
function screenshotName() {
|
||||
const d = new Date();
|
||||
const p = (n) => String(n).padStart(2, '0');
|
||||
return `Screenshot ${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}.${p(d.getMinutes())}.${p(d.getSeconds())}.png`;
|
||||
}
|
||||
|
||||
async function addFile(name, blob) {
|
||||
const bytes = new Uint8Array(await blob.arrayBuffer());
|
||||
const [np, nn] = toWasm(encoder.encode(name));
|
||||
const [dp, dn] = toWasm(bytes);
|
||||
X.web_add_file(np, nn, dp, dn);
|
||||
requestFrame();
|
||||
}
|
||||
|
||||
let dbPromise = null;
|
||||
function idb(mode, action) {
|
||||
if (!dbPromise) {
|
||||
dbPromise = new Promise((resolve, reject) => {
|
||||
const request = indexedDB.open('taskforge', 1);
|
||||
request.onupgradeneeded = () => request.result.createObjectStore('blobs');
|
||||
request.onsuccess = () => resolve(request.result);
|
||||
request.onerror = () => reject(request.error);
|
||||
});
|
||||
}
|
||||
return dbPromise.then((db) => new Promise((resolve, reject) => {
|
||||
const request = action(db.transaction('blobs', mode).objectStore('blobs'));
|
||||
request.onsuccess = () => resolve(request.result);
|
||||
request.onerror = () => reject(request.error);
|
||||
}));
|
||||
}
|
||||
|
||||
const mediaInput = document.createElement('input');
|
||||
mediaInput.type = 'file';
|
||||
mediaInput.multiple = true;
|
||||
mediaInput.accept = 'image/*,video/*';
|
||||
mediaInput.addEventListener('change', () => {
|
||||
for (const file of mediaInput.files) addFile(file.name, file);
|
||||
mediaInput.value = '';
|
||||
});
|
||||
|
||||
// Report any import we forgot instead of failing to instantiate.
|
||||
const imports = {
|
||||
env: new Proxy(env, {
|
||||
@@ -203,7 +438,9 @@ function frame(timeMs) {
|
||||
canvas.width = pw;
|
||||
canvas.height = ph;
|
||||
}
|
||||
if (X.web_frame(w, h, dpr, timeMs / 1000)) requestFrame();
|
||||
const now = Date.now() / 1000;
|
||||
const utcOffset = -new Date().getTimezoneOffset() * 60;
|
||||
if (X.web_frame(w, h, dpr, timeMs / 1000, now, utcOffset)) requestFrame();
|
||||
}
|
||||
|
||||
function sendString(text, exportName) {
|
||||
@@ -248,6 +485,13 @@ function installInput() {
|
||||
X.web_mouse_button(MOUSE_BUTTONS[e.button] ?? 0, 0);
|
||||
});
|
||||
canvas.addEventListener('contextmenu', (e) => e.preventDefault());
|
||||
// Drag & drop images and videos onto the window.
|
||||
window.addEventListener('dragover', (e) => { e.preventDefault(); });
|
||||
window.addEventListener('drop', (e) => {
|
||||
e.preventDefault();
|
||||
mouseMove(e);
|
||||
for (const f of e.dataTransfer?.files || []) addFile(f.name, f);
|
||||
});
|
||||
canvas.addEventListener('wheel', (e) => {
|
||||
const scale = e.deltaMode === 1 ? 1 : e.deltaMode === 2 ? 10 : 1 / 40;
|
||||
X.web_modifiers(modifiers(e));
|
||||
@@ -278,6 +522,13 @@ function installInput() {
|
||||
});
|
||||
window.addEventListener('keyup', (e) => { X.web_modifiers(modifiers(e)); });
|
||||
window.addEventListener('paste', (e) => {
|
||||
// Pasted images (e.g. a screenshot) become attachments; otherwise paste the text.
|
||||
const files = [...(e.clipboardData?.files || [])].filter((f) => f.type.startsWith('image/') || f.type.startsWith('video/'));
|
||||
if (files.length) {
|
||||
for (const f of files) addFile(f.name && f.name !== 'image.png' ? f.name : screenshotName(), f);
|
||||
e.preventDefault();
|
||||
return;
|
||||
}
|
||||
const text = e.clipboardData?.getData('text') || '';
|
||||
for (const ch of text.replace(/\r\n/g, '\n')) X.web_char(ch.codePointAt(0));
|
||||
e.preventDefault();
|
||||
|
||||
Reference in New Issue
Block a user