Task #2: Skill mit konfigurierter KI tatsächlich semantisch analysieren

Verified the AI analysis end-to-end with a real provider and fixed two gaps
found during the live run.

Findings & fixes:
- gpt-5 series (Replit AI Integrations modelfarm default) rejected the hardcoded
  `temperature: 0.1` with HTTP 400, silently disabling AI analysis. Removed the
  temperature param from the OpenAI-compatible request for broad model
  compatibility (aiAnalysis.ts).
- Per-rule AI config (enable/disable/severity) was only a global on/off gate and
  AI findings weren't mapped to the AI rule IDs, so individual rule severity was
  ignored. runAiAnalysis now receives the enabled AI rules, instructs the model
  to classify each finding into one of those ruleIds, drops findings for
  disabled rules, and overrides severity/axis with the configured values
  (aiAnalysis.ts + scanEngine.ts).

End-to-end verification (Replit OpenAI integration, gpt-5-mini provider):
- "KI-Analyse aktivieren" produces AI findings mapped to AI-PROMPT-INJECTION,
  AI-MALICIOUS-INTENT, AI-DATA-PRIVACY.
- Disabling AI-MALICIOUS-INTENT removed its finding; setting AI-PROMPT-INJECTION
  to critical was reflected in the result.
- Wrong baseUrl and invalid token (real OpenAI endpoint) produce understandable
  aiError messages with no token leak.

Side effects / notes:
- Set up the Replit OpenAI AI Integration (env vars) and created one enabled
  provider row ("Replit OpenAI") so AI analysis works out of the box. Each
  AI-enabled scan bills the user's Replit credits.
- Test scans created during verification were deleted.
- artifacts/api-server typecheck passes.

Replit-Task-Id: 7321caa4-5079-4db7-8ed2-4ccaa74fa577
This commit is contained in:
amertensreplit 2026-06-10 13:56:15 +00:00
parent 8eae5f4fe6
commit 9f7b67972f
4 changed files with 84 additions and 15 deletions

View file

@ -1 +1,2 @@
- [lucide-react icon name collisions](lucide-icon-name-collisions.md) — `Badge`/`Activity` from lucide collide with shadcn/ui Badge and React 19 Activity; import Badge from ui, Activity from lucide. - [lucide-react icon name collisions](lucide-icon-name-collisions.md) — `Badge`/`Activity` from lucide collide with shadcn/ui Badge and React 19 Activity; import Badge from ui, Activity from lucide.
- [OpenAI gpt-5 temperature](openai-temperature-gpt5.md) — gpt-5* reject `temperature != 1`; omit temperature in OpenAI-compatible clients or AI analysis silently fails.

View file

@ -0,0 +1,21 @@
---
name: OpenAI gpt-5 series rejects non-default temperature
description: gpt-5* models error on temperature != 1; omit temperature in OpenAI-compatible clients for cross-model compatibility
---
# OpenAI-compatible client: do not send a non-default `temperature`
When calling OpenAI-compatible `chat/completions`, the gpt-5 series (e.g. gpt-5-mini,
the Replit AI Integrations modelfarm default) returns `HTTP 400 unsupported_value`:
"`temperature` does not support 0.1 with this model. Only the default (1) value is
supported." This silently disables AI analysis (`aiUsed:false`, `aiError` set).
**Rule:** omit `temperature` from the request body for broad model compatibility,
rather than hardcoding a low value for determinism.
**Why:** older models accepted `temperature: 0.1` but gpt-5* only accept the default.
A hardcoded low temperature breaks every scan against modern models.
**How to apply:** in `artifacts/api-server/src/lib/aiAnalysis.ts` the OpenAI-compatible
path no longer sends `temperature`. If reintroducing sampling controls, gate them per
model or make them optional.

View file

@ -4,6 +4,14 @@ import type { ParsedFile, RawFinding, Severity, Axis } from "./ruleCatalog";
const SEVERITIES: Severity[] = ["critical", "high", "medium", "low", "info"]; const SEVERITIES: Severity[] = ["critical", "high", "medium", "low", "info"];
const AXES: Axis[] = ["security", "privacy"]; const AXES: Axis[] = ["security", "privacy"];
export type AiRuleConfig = {
ruleId: string;
title: string;
description: string;
axis: Axis;
severity: Severity;
};
export type AiResult = { export type AiResult = {
findings: RawFinding[]; findings: RawFinding[];
error: string | null; error: string | null;
@ -51,19 +59,24 @@ function buildSkillPayload(files: ParsedFile[]): string {
return parts.join("\n"); return parts.join("\n");
} }
function coerceFinding(raw: unknown): RawFinding | null { function coerceFinding(
raw: unknown,
allowed: Map<string, AiRuleConfig>,
): RawFinding | null {
if (!raw || typeof raw !== "object") return null; if (!raw || typeof raw !== "object") return null;
const o = raw as Record<string, unknown>; const o = raw as Record<string, unknown>;
const axis = AXES.includes(o.axis as Axis) ? (o.axis as Axis) : "security";
const severity = SEVERITIES.includes(o.severity as Severity)
? (o.severity as Severity)
: "medium";
const title = typeof o.title === "string" ? o.title.slice(0, 200) : null; const title = typeof o.title === "string" ? o.title.slice(0, 200) : null;
if (!title) return null; if (!title) return null;
const rule = typeof o.ruleId === "string" ? allowed.get(o.ruleId) : undefined;
if (!rule) {
return null;
}
return { return {
ruleId: typeof o.ruleId === "string" ? o.ruleId : "AI-FINDING", ruleId: rule.ruleId,
axis, axis: rule.axis,
severity, severity: rule.severity,
title, title,
description: description:
typeof o.description === "string" ? o.description.slice(0, 2000) : title, typeof o.description === "string" ? o.description.slice(0, 2000) : title,
@ -106,7 +119,6 @@ async function callOpenAiCompatible(
{ role: "system", content: system }, { role: "system", content: system },
{ role: "user", content: user }, { role: "user", content: user },
], ],
temperature: 0.1,
}), }),
}); });
if (!res.ok) { if (!res.ok) {
@ -163,22 +175,39 @@ export async function callProvider(
return callOpenAiCompatible(provider, system, user); return callOpenAiCompatible(provider, system, user);
} }
function buildRuleMenu(aiRules: AiRuleConfig[]): string {
const lines = aiRules.map(
(r) => `- ${r.ruleId} (${r.axis}): ${r.title}${r.description}`,
);
return [
"",
"Ordne jeden Befund GENAU EINER der folgenden aktiven Kategorien zu und gib deren Kennung im Pflichtfeld \"ruleId\" zurück. Verwende ausschließlich diese Kennungen:",
...lines,
'Befunde, die zu keiner dieser Kategorien passen, lasse weg. Das Feld "severity" wird serverseitig festgelegt und kann von dir ignoriert werden.',
].join("\n");
}
export async function runAiAnalysis( export async function runAiAnalysis(
provider: AiProvider, provider: AiProvider,
prompts: Prompt[], prompts: Prompt[],
files: ParsedFile[], files: ParsedFile[],
aiRules: AiRuleConfig[],
): Promise<AiResult> { ): Promise<AiResult> {
if (aiRules.length === 0) {
return { findings: [], error: null };
}
const allowed = new Map(aiRules.map((r) => [r.ruleId, r]));
const systemPrompt = prompts.find((p) => p.key === "system")?.content ?? ""; const systemPrompt = prompts.find((p) => p.key === "system")?.content ?? "";
const analysisPrompt = const analysisPrompt =
prompts.find((p) => p.key === "analysis")?.content ?? ""; prompts.find((p) => p.key === "analysis")?.content ?? "";
const payload = buildSkillPayload(files); const payload = buildSkillPayload(files);
const user = `${analysisPrompt}\n\nHier ist das zu prüfende Skill:\n${payload}`; const user = `${analysisPrompt}\n${buildRuleMenu(aiRules)}\n\nHier ist das zu prüfende Skill:\n${payload}`;
try { try {
const content = await callProvider(provider, systemPrompt, user); const content = await callProvider(provider, systemPrompt, user);
const parsed = extractJson(content) as { findings?: unknown[] }; const parsed = extractJson(content) as { findings?: unknown[] };
const findingsRaw = Array.isArray(parsed.findings) ? parsed.findings : []; const findingsRaw = Array.isArray(parsed.findings) ? parsed.findings : [];
const findings = findingsRaw const findings = findingsRaw
.map(coerceFinding) .map((f) => coerceFinding(f, allowed))
.filter((f): f is RawFinding => f !== null); .filter((f): f is RawFinding => f !== null);
return { findings, error: null }; return { findings, error: null };
} catch (err) { } catch (err) {

View file

@ -8,13 +8,15 @@ import {
import { eq } from "drizzle-orm"; import { eq } from "drizzle-orm";
import { import {
STATIC_RULES, STATIC_RULES,
AI_RULES,
runStaticRule, runStaticRule,
type ParsedFile, type ParsedFile,
type RawFinding, type RawFinding,
type Severity, type Severity,
type Axis,
} from "./ruleCatalog"; } from "./ruleCatalog";
import type { FindingCounts as DbFindingCounts } from "@workspace/db"; import type { FindingCounts as DbFindingCounts } from "@workspace/db";
import { runAiAnalysis } from "./aiAnalysis"; import { runAiAnalysis, type AiRuleConfig } from "./aiAnalysis";
const SEVERITY_WEIGHT: Record<Severity, number> = { const SEVERITY_WEIGHT: Record<Severity, number> = {
critical: 50, critical: 50,
@ -91,8 +93,19 @@ export async function analyzeSkill(
let aiError: string | null = null; let aiError: string | null = null;
if (useAi) { if (useAi) {
const aiRuleIds = new Set(AI_RULES.map((r) => r.ruleId));
const enabledAiRules: AiRuleConfig[] = AI_RULES.filter((rule) => {
const cfg = ruleConfig.get(rule.ruleId);
return cfg ? cfg.enabled : true;
}).map((rule) => ({
ruleId: rule.ruleId,
title: rule.title,
description: rule.description,
axis: rule.axis as Axis,
severity: ruleConfig.get(rule.ruleId)?.severity ?? rule.defaultSeverity,
}));
const aiRulesEnabled = dbRules const aiRulesEnabled = dbRules
.filter((r) => r.detectionType === "ai") .filter((r) => r.detectionType === "ai" || aiRuleIds.has(r.ruleId))
.some((r) => r.enabled); .some((r) => r.enabled);
const [provider] = await db const [provider] = await db
.select() .select()
@ -100,7 +113,7 @@ export async function analyzeSkill(
.where(eq(aiProvidersTable.enabled, true)) .where(eq(aiProvidersTable.enabled, true))
.limit(1); .limit(1);
if (!aiRulesEnabled) { if (!aiRulesEnabled || enabledAiRules.length === 0) {
aiError = "KI-Regeln sind im Regelwerk deaktiviert."; aiError = "KI-Regeln sind im Regelwerk deaktiviert.";
} else if (!provider) { } else if (!provider) {
aiError = aiError =
@ -109,7 +122,12 @@ export async function analyzeSkill(
aiError = `Für den Provider "${provider.name}" ist kein API-Token hinterlegt.`; aiError = `Für den Provider "${provider.name}" ist kein API-Token hinterlegt.`;
} else { } else {
const prompts: Prompt[] = await db.select().from(promptsTable); const prompts: Prompt[] = await db.select().from(promptsTable);
const result = await runAiAnalysis(provider, prompts, files); const result = await runAiAnalysis(
provider,
prompts,
files,
enabledAiRules,
);
aiError = result.error; aiError = result.error;
if (!result.error) { if (!result.error) {
aiUsed = true; aiUsed = true;