diff --git a/.agents/memory/MEMORY.md b/.agents/memory/MEMORY.md index 55b5f35..5793f2b 100644 --- a/.agents/memory/MEMORY.md +++ b/.agents/memory/MEMORY.md @@ -1 +1,2 @@ - [lucide-react icon name collisions](lucide-icon-name-collisions.md) — `Badge`/`Activity` from lucide collide with shadcn/ui Badge and React 19 Activity; import Badge from ui, Activity from lucide. +- [OpenAI gpt-5 temperature](openai-temperature-gpt5.md) — gpt-5* reject `temperature != 1`; omit temperature in OpenAI-compatible clients or AI analysis silently fails. diff --git a/.agents/memory/openai-temperature-gpt5.md b/.agents/memory/openai-temperature-gpt5.md new file mode 100644 index 0000000..625c1b8 --- /dev/null +++ b/.agents/memory/openai-temperature-gpt5.md @@ -0,0 +1,21 @@ +--- +name: OpenAI gpt-5 series rejects non-default temperature +description: gpt-5* models error on temperature != 1; omit temperature in OpenAI-compatible clients for cross-model compatibility +--- + +# OpenAI-compatible client: do not send a non-default `temperature` + +When calling OpenAI-compatible `chat/completions`, the gpt-5 series (e.g. gpt-5-mini, +the Replit AI Integrations modelfarm default) returns `HTTP 400 unsupported_value`: +"`temperature` does not support 0.1 with this model. Only the default (1) value is +supported." This silently disables AI analysis (`aiUsed:false`, `aiError` set). + +**Rule:** omit `temperature` from the request body for broad model compatibility, +rather than hardcoding a low value for determinism. + +**Why:** older models accepted `temperature: 0.1` but gpt-5* only accept the default. +A hardcoded low temperature breaks every scan against modern models. + +**How to apply:** in `artifacts/api-server/src/lib/aiAnalysis.ts` the OpenAI-compatible +path no longer sends `temperature`. If reintroducing sampling controls, gate them per +model or make them optional. diff --git a/artifacts/api-server/src/lib/aiAnalysis.ts b/artifacts/api-server/src/lib/aiAnalysis.ts index f355afd..237f6e2 100644 --- a/artifacts/api-server/src/lib/aiAnalysis.ts +++ b/artifacts/api-server/src/lib/aiAnalysis.ts @@ -4,6 +4,14 @@ import type { ParsedFile, RawFinding, Severity, Axis } from "./ruleCatalog"; const SEVERITIES: Severity[] = ["critical", "high", "medium", "low", "info"]; const AXES: Axis[] = ["security", "privacy"]; +export type AiRuleConfig = { + ruleId: string; + title: string; + description: string; + axis: Axis; + severity: Severity; +}; + export type AiResult = { findings: RawFinding[]; error: string | null; @@ -51,19 +59,24 @@ function buildSkillPayload(files: ParsedFile[]): string { return parts.join("\n"); } -function coerceFinding(raw: unknown): RawFinding | null { +function coerceFinding( + raw: unknown, + allowed: Map, +): RawFinding | null { if (!raw || typeof raw !== "object") return null; const o = raw as Record; - const axis = AXES.includes(o.axis as Axis) ? (o.axis as Axis) : "security"; - const severity = SEVERITIES.includes(o.severity as Severity) - ? (o.severity as Severity) - : "medium"; const title = typeof o.title === "string" ? o.title.slice(0, 200) : null; if (!title) return null; + + const rule = typeof o.ruleId === "string" ? allowed.get(o.ruleId) : undefined; + if (!rule) { + return null; + } + return { - ruleId: typeof o.ruleId === "string" ? o.ruleId : "AI-FINDING", - axis, - severity, + ruleId: rule.ruleId, + axis: rule.axis, + severity: rule.severity, title, description: typeof o.description === "string" ? o.description.slice(0, 2000) : title, @@ -106,7 +119,6 @@ async function callOpenAiCompatible( { role: "system", content: system }, { role: "user", content: user }, ], - temperature: 0.1, }), }); if (!res.ok) { @@ -163,22 +175,39 @@ export async function callProvider( return callOpenAiCompatible(provider, system, user); } +function buildRuleMenu(aiRules: AiRuleConfig[]): string { + const lines = aiRules.map( + (r) => `- ${r.ruleId} (${r.axis}): ${r.title} — ${r.description}`, + ); + return [ + "", + "Ordne jeden Befund GENAU EINER der folgenden aktiven Kategorien zu und gib deren Kennung im Pflichtfeld \"ruleId\" zurück. Verwende ausschließlich diese Kennungen:", + ...lines, + 'Befunde, die zu keiner dieser Kategorien passen, lasse weg. Das Feld "severity" wird serverseitig festgelegt und kann von dir ignoriert werden.', + ].join("\n"); +} + export async function runAiAnalysis( provider: AiProvider, prompts: Prompt[], files: ParsedFile[], + aiRules: AiRuleConfig[], ): Promise { + if (aiRules.length === 0) { + return { findings: [], error: null }; + } + const allowed = new Map(aiRules.map((r) => [r.ruleId, r])); const systemPrompt = prompts.find((p) => p.key === "system")?.content ?? ""; const analysisPrompt = prompts.find((p) => p.key === "analysis")?.content ?? ""; const payload = buildSkillPayload(files); - const user = `${analysisPrompt}\n\nHier ist das zu prüfende Skill:\n${payload}`; + const user = `${analysisPrompt}\n${buildRuleMenu(aiRules)}\n\nHier ist das zu prüfende Skill:\n${payload}`; try { const content = await callProvider(provider, systemPrompt, user); const parsed = extractJson(content) as { findings?: unknown[] }; const findingsRaw = Array.isArray(parsed.findings) ? parsed.findings : []; const findings = findingsRaw - .map(coerceFinding) + .map((f) => coerceFinding(f, allowed)) .filter((f): f is RawFinding => f !== null); return { findings, error: null }; } catch (err) { diff --git a/artifacts/api-server/src/lib/scanEngine.ts b/artifacts/api-server/src/lib/scanEngine.ts index f3ebb5f..377f1e7 100644 --- a/artifacts/api-server/src/lib/scanEngine.ts +++ b/artifacts/api-server/src/lib/scanEngine.ts @@ -8,13 +8,15 @@ import { import { eq } from "drizzle-orm"; import { STATIC_RULES, + AI_RULES, runStaticRule, type ParsedFile, type RawFinding, type Severity, + type Axis, } from "./ruleCatalog"; import type { FindingCounts as DbFindingCounts } from "@workspace/db"; -import { runAiAnalysis } from "./aiAnalysis"; +import { runAiAnalysis, type AiRuleConfig } from "./aiAnalysis"; const SEVERITY_WEIGHT: Record = { critical: 50, @@ -91,8 +93,19 @@ export async function analyzeSkill( let aiError: string | null = null; if (useAi) { + const aiRuleIds = new Set(AI_RULES.map((r) => r.ruleId)); + const enabledAiRules: AiRuleConfig[] = AI_RULES.filter((rule) => { + const cfg = ruleConfig.get(rule.ruleId); + return cfg ? cfg.enabled : true; + }).map((rule) => ({ + ruleId: rule.ruleId, + title: rule.title, + description: rule.description, + axis: rule.axis as Axis, + severity: ruleConfig.get(rule.ruleId)?.severity ?? rule.defaultSeverity, + })); const aiRulesEnabled = dbRules - .filter((r) => r.detectionType === "ai") + .filter((r) => r.detectionType === "ai" || aiRuleIds.has(r.ruleId)) .some((r) => r.enabled); const [provider] = await db .select() @@ -100,7 +113,7 @@ export async function analyzeSkill( .where(eq(aiProvidersTable.enabled, true)) .limit(1); - if (!aiRulesEnabled) { + if (!aiRulesEnabled || enabledAiRules.length === 0) { aiError = "KI-Regeln sind im Regelwerk deaktiviert."; } else if (!provider) { aiError = @@ -109,7 +122,12 @@ export async function analyzeSkill( aiError = `Für den Provider "${provider.name}" ist kein API-Token hinterlegt.`; } else { const prompts: Prompt[] = await db.select().from(promptsTable); - const result = await runAiAnalysis(provider, prompts, files); + const result = await runAiAnalysis( + provider, + prompts, + files, + enabledAiRules, + ); aiError = result.error; if (!result.error) { aiUsed = true;