feat(files): mship file writing improvements- #6933 (#6933)

feat(files): mship file writing improvements (#6933)

Author: Sg312

apps/sim/lib/copilot/tools/server/files/doc-compile.ts

@@ -8,6 +8,7 @@ import {
   publishCompiledDocArtifact,
   storeCompiledDoc,
 } from '@/lib/copilot/tools/server/files/doc-compiled-store'
+import { PPTX_SHIM_JS } from '@/lib/copilot/tools/server/files/pptx-shim'
 import { isDocSandboxEnabled } from '@/lib/core/config/env-flags'
 import { CodeLanguage } from '@/lib/execution/languages'
 import {
@@ -103,7 +104,20 @@ export async function getE2BDocFormat(fileName: string): Promise<E2BDocFormat |
 // "file not staged" and every workspace-image embed silently fails.
 const INPUT_PATH_RE = /\/home\/user\/inputs\/([A-Za-z0-9_-]+)/g
 const FILE_HELPER_RE =
-  /\b(?:getFileBase64|addImage|drawImage)\(\s*(?:[A-Za-z_$][\w$]*\s*,\s*)?['"]([A-Za-z0-9_-]+)['"]/g
+  /\b(?:getFileBase64|addImage|drawImage|input_path)\(\s*(?:[A-Za-z_$][\w$]*\s*,\s*)?['"]([A-Za-z0-9_-]+)['"]/g
+
+/**
+ * A .pptx or .docx source whose first line is `#!simdoc` is a template-clone
+ * script: Python against the simdoc Deck/Doc API (opening a retained reference
+ * staged via `input_path('wf_…')`), compiled by the Python engine instead of
+ * the Node engine. Only meaningful when the doc sandbox is enabled — the
+ * legacy isolated-vm path has no Python and refuses these sources explicitly.
+ */
+const SIMDOC_DECK_MARKER = '#!simdoc'
+
+export function isSimdocDeckSource(source: string): boolean {
+  return source.trimStart().startsWith(SIMDOC_DECK_MARKER)
+}
 
 // The doc source is user/LLM-controlled, so bound how much it can pull into the
 // sandbox by BYTES (per file and total) — an authenticated member must not be
@@ -372,6 +386,7 @@ function __mime(b){ if(b.length>=2&&b[0]===0x89&&b[1]===0x50)return 'image/png';
 globalThis.getFileBase64 = async function(fileId){ const p='/home/user/inputs/'+fileId; if(!fs.existsSync(p)) throw new Error('getFileBase64: file not staged: '+fileId); const b=fs.readFileSync(p); return __mime(b)+';base64,'+b.toString('base64'); };
 globalThis.addImage = async function(slide, fileId, opts){ if(!opts||opts.x==null||opts.y==null||opts.w==null||opts.h==null) throw new Error('addImage: opts must include x, y, w, h'); const data=await globalThis.getFileBase64(fileId); slide.addImage(Object.assign({}, opts, { data })); };
 globalThis.iconImage = async function(IconComponent, color, size){ const React=require('react'); const RDS=require('react-dom/server'); const sharp=require('sharp'); const svg=RDS.renderToStaticMarkup(React.createElement(IconComponent,{color:color||'#000000',size:String(size||256)})); const png=await sharp(Buffer.from(svg)).png().toBuffer(); return 'image/png;base64,'+png.toString('base64'); };
+${PPTX_SHIM_JS}
 `.trim()
 
 const DOCX_NODE_PREAMBLE = `
@@ -417,8 +432,22 @@ ${finalize}
 })().then(() => console.log('__DOC_OK__')).catch((e) => { console.error('__DOC_ERR__' + (e && e.message ? e.message : String(e))); process.exit(1); });
 `
 
+  // After a successful build, run simdoc's structural validation in the same
+  // sandbox call. It catches the defect classes Office rejects or silently
+  // discards (chart axis faults, stacked-label positions, broken
+  // relationships) that a successful compile and a clean render both miss.
+  // Best-effort by construction: on an image built before simdoc existed the
+  // command produces no sentinel and the compile proceeds unvalidated.
+  const command = `NODE_PATH=$(npm root -g) node /home/user/script.js
+__doc_status=$?
+if [ $__doc_status -eq 0 ] && [ -f /home/user/output.${ext} ]; then
+  __simdoc_report=$(python3 -m simdoc validate /home/user/output.${ext} 2>/dev/null | tr '\\n' ' ')
+  if [ -n "$__simdoc_report" ]; then echo "__SIMDOC_VALIDATE__$__simdoc_report"; fi
+fi
+exit $__doc_status`
+
   const result = await executeShellInSandbox({
-    code: 'NODE_PATH=$(npm root -g) node /home/user/script.js',
+    code: command,
     envs: {},
     timeoutMs: DOC_COMPILE_TIMEOUT_MS,
     sandboxKind: 'doc',
@@ -439,6 +468,7 @@ ${finalize}
   const out = `${result.stdout || ''}\n${result.error || ''}`
   const errMatch = out.match(/__DOC_ERR__([\s\S]*)/)
   if (out.includes('__DOC_OK__') && result.exportedFileContent) {
+    assertSimdocValidationPassed(out, ext)
     return Buffer.from(result.exportedFileContent, 'base64')
   }
   if (errMatch) {
@@ -456,14 +486,64 @@ ${finalize}
   )
 }
 
+interface SimdocIssue {
+  code?: string
+  part?: string
+  message?: string
+  fix?: string
+}
+
+const MAX_REPORTED_VALIDATION_ISSUES = 10
+
+/**
+ * Parses the __SIMDOC_VALIDATE__ sentinel a pptx compile emits and throws a
+ * DocCompileUserError when the built deck failed structural validation. A
+ * missing or unparseable sentinel means the toolkit is absent or misbehaved —
+ * that degrades to an unvalidated compile, never a failed one.
+ */
+function assertSimdocValidationPassed(compileOutput: string, ext: string): void {
+  const sentinel = compileOutput.match(/__SIMDOC_VALIDATE__(.*)/)
+  if (!sentinel?.[1]) return
+  let report: { ok?: boolean; issues?: SimdocIssue[] }
+  try {
+    report = JSON.parse(sentinel[1].trim())
+  } catch {
+    logger.warn('simdoc validation output was not parseable; compile proceeds unvalidated')
+    return
+  }
+  if (report.ok !== false || !Array.isArray(report.issues) || report.issues.length === 0) return
+  const lines = report.issues.slice(0, MAX_REPORTED_VALIDATION_ISSUES).map((issue) => {
+    const location = issue.part ? ` ${issue.part}:` : ''
+    const fix = issue.fix ? ` Fix: ${issue.fix}.` : ''
+    return `- [${issue.code ?? 'issue'}]${location} ${issue.message ?? 'unknown'}.${fix}`
+  })
+  const extra =
+    report.issues.length > lines.length ? `\n(+${report.issues.length - lines.length} more)` : ''
+  const app = ext === 'docx' ? 'Word' : 'PowerPoint'
+  throw new DocCompileUserError(
+    `${ext.toUpperCase()} structural validation failed — ${app} would reject or silently discard content in this file. Fix the source and retry:\n${lines.join('\n')}${extra}`
+  )
+}
+
 async function buildCompiledDoc(
   args: CompileArgs,
   fmt: E2BDocFormat,
   referencedImages: ReferencedImageResolution
 ): Promise<CompiledDocResult> {
   const { source, fileName, workspaceId, filePrincipal } = args
-  const buffer =
-    fmt.engine === 'node'
+  const cloneDeck = (fmt.ext === 'pptx' || fmt.ext === 'docx') && isSimdocDeckSource(source)
+  const buffer = cloneDeck
+    ? await compileDocViaE2BPython(
+        {
+          source: wrapSimdocDeckSource(source, fmt.ext as 'pptx' | 'docx'),
+          fileName,
+          workspaceId,
+          filePrincipal,
+        },
+        fmt,
+        referencedImages
+      )
+    : fmt.engine === 'node'
       ? await compileDocViaE2BNode(
           { source, fileName, workspaceId, filePrincipal },
           fmt.ext as 'pptx' | 'docx',
@@ -490,6 +570,57 @@ async function buildCompiledDoc(
   }
 }
 
+// Template-clone scripts author against the simdoc Deck (pptx) / Doc (docx)
+// API. The prelude supplies input_path (staged workspace files) and
+// OUTPUT_PATH; the finalizer saves the expected variable (scripts never save
+// themselves, mirroring the JS engines' finalizers) and then structurally
+// validates the result in-process. The validation import degrades on images
+// built before simdoc existed.
+function simdocPrelude(ext: 'pptx' | 'docx'): string {
+  return `
+import os as __sim_os
+OUTPUT_PATH = '/home/user/output.${ext}'
+def input_path(file_id):
+    __p = '/home/user/inputs/' + file_id
+    if not __sim_os.path.exists(__p):
+        raise FileNotFoundError(
+            'input_path: file not staged: ' + file_id +
+            ' (pass the workspace file id as a string literal at the call site)'
+        )
+    return __p
+`.trim()
+}
+
+function simdocFinalize(ext: 'pptx' | 'docx'): string {
+  const variable = ext === 'pptx' ? 'deck' : 'doc'
+  const className = ext === 'pptx' ? 'Deck' : 'Doc'
+  return `
+try:
+    ${variable}
+except NameError as __sim_err:
+    raise RuntimeError(
+        "simdoc ${ext} scripts must create a ${className} named '${variable}': ${variable} = ${className}.open(input_path('wf_...'))"
+    ) from __sim_err
+${variable}.save(OUTPUT_PATH)
+try:
+    from simdoc.validate import validate_file as __sim_validate
+except ImportError:
+    __sim_validate = None
+if __sim_validate is not None:
+    __sim_report = __sim_validate(OUTPUT_PATH)
+    if not __sim_report.ok:
+        import json as __sim_json
+        raise RuntimeError(
+            'structural validation failed: '
+            + __sim_json.dumps([__i.to_dict() for __i in __sim_report.issues[:10]])
+        )
+`.trim()
+}
+
+function wrapSimdocDeckSource(source: string, ext: 'pptx' | 'docx'): string {
+  return `${simdocPrelude(ext)}\n${source}\n${simdocFinalize(ext)}`
+}
+
 interface CompilableFormat {
   magic: Buffer
   taskId: SandboxTaskId
@@ -513,6 +644,11 @@ async function compileDocInLegacySandbox(
   if (!format) {
     throw new DocCompileUserError('Document is still being generated', { pending: true })
   }
+  if ((fmt.ext === 'pptx' || fmt.ext === 'docx') && isSimdocDeckSource(args.source)) {
+    throw new DocCompileUserError(
+      'Template-clone scripts (#!simdoc) require the document sandbox, which is not enabled. Build the document with the injected JavaScript library instead.'
+    )
+  }
 
   const cacheKey = sha256Hex(`.${fmt.ext}${args.source}${args.workspaceId}`)
   const cached = compiledDocCache.get(cacheKey)