feat(risk-control): add content moderation audit
This commit is contained in:
@@ -90,6 +90,51 @@ func TestOpenAIGatewayServiceParseOpenAIImagesRequest_MultipartEdit(t *testing.T
|
||||
require.Equal(t, OpenAIImagesCapabilityNative, parsed.RequiredCapability)
|
||||
}
|
||||
|
||||
func TestOpenAIImagesRequestModerationBody_JSONEditIncludesInputImageURLs(t *testing.T) {
|
||||
parsed := &OpenAIImagesRequest{
|
||||
Endpoint: openAIImagesEditsEndpoint,
|
||||
Prompt: "replace background",
|
||||
InputImageURLs: []string{"https://example.com/source.png"},
|
||||
MaskImageURL: "https://example.com/mask.png",
|
||||
}
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIImages, parsed.ModerationBody())
|
||||
|
||||
require.Equal(t, "replace background", input.Text)
|
||||
require.Equal(t, []string{"https://example.com/source.png", "https://example.com/mask.png"}, input.Images)
|
||||
}
|
||||
|
||||
func TestOpenAIImagesRequestModerationBody_MultipartEditIncludesUploadsInMemory(t *testing.T) {
|
||||
parsed := &OpenAIImagesRequest{
|
||||
Endpoint: openAIImagesEditsEndpoint,
|
||||
Prompt: "replace background",
|
||||
Uploads: []OpenAIImagesUpload{{
|
||||
FieldName: "image",
|
||||
FileName: "source.png",
|
||||
ContentType: "image/png",
|
||||
Data: []byte("fake-image-bytes"),
|
||||
}},
|
||||
MaskUpload: &OpenAIImagesUpload{
|
||||
FieldName: "mask",
|
||||
FileName: "mask.png",
|
||||
ContentType: "image/png",
|
||||
Data: []byte("fake-mask-bytes"),
|
||||
},
|
||||
}
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIImages, parsed.ModerationBody())
|
||||
|
||||
require.Equal(t, "replace background", input.Text)
|
||||
require.Equal(t, []string{
|
||||
"data:image/png;base64,ZmFrZS1pbWFnZS1ieXRlcw==",
|
||||
"data:image/png;base64,ZmFrZS1tYXNrLWJ5dGVz",
|
||||
}, input.Images)
|
||||
|
||||
log := (&ContentModerationService{}).buildLog(ContentModerationCheckInput{}, defaultContentModerationConfig(), ContentModerationActionAllow, false, "", 0, nil, input.ExcerptText(), nil, nil, "")
|
||||
require.Equal(t, "replace background", log.InputExcerpt)
|
||||
require.NotContains(t, log.InputExcerpt, "ZmFrZS")
|
||||
}
|
||||
|
||||
func TestOpenAIGatewayServiceParseOpenAIImagesRequest_NormalizesOfficialAndCustomSizes(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user