feat(risk-control): add content moderation audit

This commit is contained in:
shaw
2026-05-07 09:14:47 +08:00
parent a1106e8167
commit fff4a300c6
54 changed files with 6840 additions and 34 deletions
@@ -90,6 +90,51 @@ func TestOpenAIGatewayServiceParseOpenAIImagesRequest_MultipartEdit(t *testing.T
require.Equal(t, OpenAIImagesCapabilityNative, parsed.RequiredCapability)
}
func TestOpenAIImagesRequestModerationBody_JSONEditIncludesInputImageURLs(t *testing.T) {
parsed := &OpenAIImagesRequest{
Endpoint: openAIImagesEditsEndpoint,
Prompt: "replace background",
InputImageURLs: []string{"https://example.com/source.png"},
MaskImageURL: "https://example.com/mask.png",
}
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIImages, parsed.ModerationBody())
require.Equal(t, "replace background", input.Text)
require.Equal(t, []string{"https://example.com/source.png", "https://example.com/mask.png"}, input.Images)
}
func TestOpenAIImagesRequestModerationBody_MultipartEditIncludesUploadsInMemory(t *testing.T) {
parsed := &OpenAIImagesRequest{
Endpoint: openAIImagesEditsEndpoint,
Prompt: "replace background",
Uploads: []OpenAIImagesUpload{{
FieldName: "image",
FileName: "source.png",
ContentType: "image/png",
Data: []byte("fake-image-bytes"),
}},
MaskUpload: &OpenAIImagesUpload{
FieldName: "mask",
FileName: "mask.png",
ContentType: "image/png",
Data: []byte("fake-mask-bytes"),
},
}
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIImages, parsed.ModerationBody())
require.Equal(t, "replace background", input.Text)
require.Equal(t, []string{
"data:image/png;base64,ZmFrZS1pbWFnZS1ieXRlcw==",
"data:image/png;base64,ZmFrZS1tYXNrLWJ5dGVz",
}, input.Images)
log := (&ContentModerationService{}).buildLog(ContentModerationCheckInput{}, defaultContentModerationConfig(), ContentModerationActionAllow, false, "", 0, nil, input.ExcerptText(), nil, nil, "")
require.Equal(t, "replace background", log.InputExcerpt)
require.NotContains(t, log.InputExcerpt, "ZmFrZS")
}
func TestOpenAIGatewayServiceParseOpenAIImagesRequest_NormalizesOfficialAndCustomSizes(t *testing.T) {
gin.SetMode(gin.TestMode)