Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/client/src/widgets/import/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -151,14 +151,14 @@ const InitialDialog = ({ setModal, successAction }: InitialDialogProps) => {
`}
>
<FiInfo className={tw`mr-1.5 inline-block size-4 align-bottom`} />
Import Postman or HAR files
Import Postman, HAR, Swagger or OpenAPI files, or paste a URL
</div>

<TextInputField
className={tw`mt-4`}
label='Text value'
onChange={setText}
placeholder='Paste cURL, Raw text or URL...'
placeholder='Paste cURL, Swagger/OpenAPI URL, or raw text...'
value={text}
/>

Expand Down
117 changes: 109 additions & 8 deletions packages/server/internal/api/rimportv2/format_detection.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const (
FormatJSON
FormatCURL
FormatPostman
FormatOpenAPI
)

const ReasonValidJSON = "Valid JSON; "
Expand All @@ -42,6 +43,8 @@ func (f Format) String() string {
return "CURL"
case FormatPostman:
return "Postman"
case FormatOpenAPI:
return "OpenAPI"
default:
return "Unknown"
}
Expand All @@ -57,19 +60,27 @@ type DetectionResult struct {
// FormatDetector implements automatic format detection with confidence scoring
type FormatDetector struct {
// Pre-compiled regular expressions for performance
harPattern *regexp.Regexp
curlPattern *regexp.Regexp
postmanPattern *regexp.Regexp
yamlPattern *regexp.Regexp
harPattern *regexp.Regexp
curlPattern *regexp.Regexp
postmanPattern *regexp.Regexp
yamlPattern *regexp.Regexp
swaggerPattern *regexp.Regexp
openapi3Pattern *regexp.Regexp
yamlSwaggerPat *regexp.Regexp
yamlOpenapi3Pat *regexp.Regexp
}

// NewFormatDetector creates a new format detector with compiled patterns
func NewFormatDetector() *FormatDetector {
return &FormatDetector{
harPattern: regexp.MustCompile(`^\s*\{?\s*"?log"?[\s\S]*"?entries"?[\s\S]*\}?\s*$`),
curlPattern: regexp.MustCompile(`(?i)^\s*curl\s+`),
postmanPattern: regexp.MustCompile(`(?i)"?info"?\s*:\s*\{[\s\S]*"?name"?[\s\S]*"?schema"?\s*:\s*"https://schema\.getpostman\.com/json/collection/v2\.1\.0/collection\.json"`),
yamlPattern: regexp.MustCompile(`(?i)^\s*flows?\s*:`),
harPattern: regexp.MustCompile(`^\s*\{?\s*"?log"?[\s\S]*"?entries"?[\s\S]*\}?\s*$`),
curlPattern: regexp.MustCompile(`(?i)^\s*curl\s+`),
postmanPattern: regexp.MustCompile(`(?i)"?info"?\s*:\s*\{[\s\S]*"?name"?[\s\S]*"?schema"?\s*:\s*"https://schema\.getpostman\.com/json/collection/v2\.1\.0/collection\.json"`),
yamlPattern: regexp.MustCompile(`(?i)^\s*flows?\s*:`),
swaggerPattern: regexp.MustCompile(`(?i)"swagger"\s*:\s*"2\.\d+"`),
openapi3Pattern: regexp.MustCompile(`(?i)"openapi"\s*:\s*"3\.\d+\.\d+"`),
yamlSwaggerPat: regexp.MustCompile(`(?im)^swagger\s*:\s*["']?2\.\d+`),
yamlOpenapi3Pat: regexp.MustCompile(`(?im)^openapi\s*:\s*["']?3\.\d+`),
}
}

Expand All @@ -91,6 +102,7 @@ func (fd *FormatDetector) DetectFormat(data []byte) *DetectionResult {
results := []*DetectionResult{
fd.detectHAR(trimmed),
fd.detectPostman(trimmed),
fd.detectOpenAPI(trimmed),
fd.detectCURL(trimmed),
fd.detectYAML(trimmed),
fd.detectJSON(trimmed),
Expand Down Expand Up @@ -403,6 +415,93 @@ func (fd *FormatDetector) detectYAML(content string) *DetectionResult {
}
}

// detectOpenAPI detects OpenAPI/Swagger spec format with confidence scoring.
// Supports both Swagger 2.0 ("swagger": "2.0") and OpenAPI 3.x ("openapi": "3.x.x").
func (fd *FormatDetector) detectOpenAPI(content string) *DetectionResult {
confidence := 0.0
reason := ""

// Check for Swagger 2.0 pattern (JSON)
if fd.swaggerPattern.MatchString(content) {
confidence += 0.9
reason += "Swagger 2.0 spec detected; "
}

// Check for OpenAPI 3.x pattern (JSON)
if fd.openapi3Pattern.MatchString(content) {
confidence += 0.9
reason += "OpenAPI 3.x spec detected; "
}

// Check for YAML-format Swagger 2.0
if fd.yamlSwaggerPat.MatchString(content) {
confidence += 0.9
reason += "Swagger 2.0 YAML spec detected; "
}

// Check for YAML-format OpenAPI 3.x
if fd.yamlOpenapi3Pat.MatchString(content) {
confidence += 0.9
reason += "OpenAPI 3.x YAML spec detected; "
}

// Look for paths field (key indicator of OpenAPI/Swagger)
if strings.Contains(content, `"paths"`) || strings.Contains(content, "paths:") {
confidence += 0.2
reason += "paths field found; "
}

// Look for info field
if strings.Contains(content, `"info"`) || strings.Contains(content, "info:") {
confidence += 0.1
reason += "info field found; "
}

// Validate it's valid JSON or YAML
var jsonData map[string]interface{}
if err := json.Unmarshal([]byte(content), &jsonData); err == nil {
confidence += 0.1
reason += ReasonValidJSON
} else {
// Try YAML
var yamlData map[string]interface{}
if err := yaml.Unmarshal([]byte(content), &yamlData); err == nil {
confidence += 0.1
reason += "Valid YAML; "
}
}

if confidence < 0 {
confidence = 0
}

return &DetectionResult{
Format: FormatOpenAPI,
Confidence: confidence,
Reason: strings.TrimSpace(reason),
}
}

// validateOpenAPI validates OpenAPI/Swagger format specifically
func (fd *FormatDetector) validateOpenAPI(data []byte) error {
// Try JSON first, then YAML
var raw map[string]interface{}
if err := json.Unmarshal(data, &raw); err != nil {
if err := yaml.Unmarshal(data, &raw); err != nil {
return fmt.Errorf("invalid OpenAPI spec: not valid JSON or YAML: %w", err)
}
}

// Must have either "swagger" or "openapi" field
_, hasSwagger := raw["swagger"]
_, hasOpenAPI := raw["openapi"]
if !hasSwagger && !hasOpenAPI {
return fmt.Errorf("missing 'swagger' or 'openapi' version field")
}

return nil
}

// detectJSON detects generic JSON format with confidence scoring
func (fd *FormatDetector) detectJSON(content string) *DetectionResult {
confidence := 0.0
Expand Down Expand Up @@ -486,6 +585,8 @@ func (fd *FormatDetector) ValidateFormat(data []byte, format Format) error {
return fd.validateYAML(data)
case FormatJSON:
return fd.validateJSON(data)
case FormatOpenAPI:
return fd.validateOpenAPI(data)
default:
return fmt.Errorf("unknown format: %v", format)
}
Expand Down
66 changes: 64 additions & 2 deletions packages/server/internal/api/rimportv2/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ type ImportConstraints struct {
func DefaultConstraints() *ImportConstraints {
return &ImportConstraints{
MaxDataSizeBytes: 50 * 1024 * 1024, // 50MB
SupportedFormats: []Format{FormatHAR, FormatYAML, FormatJSON, FormatCURL, FormatPostman},
SupportedFormats: []Format{FormatHAR, FormatYAML, FormatJSON, FormatCURL, FormatPostman, FormatOpenAPI},
AllowedMimeTypes: []string{
"application/json",
"application/har",
Expand Down Expand Up @@ -218,6 +218,13 @@ func WithLogger(logger *slog.Logger) ServiceOption {
}
}

// WithURLFetcher sets a custom URL fetcher (useful for testing)
func WithURLFetcher(fetcher URLFetcher) ServiceOption {
return func(s *Service) {
s.urlFetcher = fetcher
}
}

// WithHTTPService sets the HTTP service for the service (required for HAR import overwrite detection)
func WithHTTPService(httpService *shttp.HTTPService) ServiceOption {
return func(s *Service) {
Expand All @@ -231,6 +238,7 @@ type Service struct {
importer Importer
validator Validator
translatorRegistry *TranslatorRegistry
urlFetcher URLFetcher
logger *slog.Logger
timeout time.Duration
}
Expand All @@ -244,6 +252,7 @@ func NewService(importer Importer, validator Validator, opts ...ServiceOption) *
importer: importer,
validator: validator,
translatorRegistry: NewTranslatorRegistry(nil), // Auto-initialize translator registry without HTTP service (will be overridden if provided)
urlFetcher: NewURLFetcher(),
timeout: 30 * time.Minute, // Default timeout for import processing
logger: slog.Default(), // Default logger
}
Expand Down Expand Up @@ -461,7 +470,11 @@ func (s *Service) ImportWithTextData(ctx context.Context, req *ImportRequest) (*
return s.Import(ctx, req)
}

// ImportUnified processes any supported format with automatic detection
// ImportUnified processes any supported format with automatic detection.
// It handles three input modes:
// 1. Binary data (req.Data) - file uploads (Postman JSON, HAR, etc.)
// 2. Text data (req.TextData) - pasted curl commands, raw JSON/YAML
// 3. URL text (req.TextData is a URL) - fetches content from URL (e.g., swagger.json URL)
func (s *Service) ImportUnified(ctx context.Context, req *ImportRequest) (*ImportResults, error) {
s.logger.Debug("ImportUnified: Starting", "workspace_id", req.WorkspaceID)
// Check if context is already cancelled
Expand Down Expand Up @@ -490,6 +503,11 @@ func (s *Service) ImportUnified(ctx context.Context, req *ImportRequest) (*Impor
return nil, err // Return the original error for workspace access issues
}

// Resolve input data: handle TextData and URL fetching
if err := s.resolveInputData(ctx, req); err != nil {
return nil, err
}

s.logger.Debug("ImportUnified: Translating data")
// Detect format and translate data
translationResult, err := s.importer.ImportAndStoreUnified(ctx, req.Data, req.WorkspaceID)
Expand Down Expand Up @@ -701,6 +719,50 @@ func (s *Service) ImportUnifiedWithTextData(ctx context.Context, req *ImportRequ
return s.ImportUnified(ctx, req)
}

// resolveInputData ensures req.Data is populated for format detection.
// It handles three cases:
// 1. Data already present (file upload) - no action needed
// 2. TextData is a URL - fetch content from the URL
// 3. TextData is raw text (curl, JSON, YAML) - convert to bytes
func (s *Service) resolveInputData(ctx context.Context, req *ImportRequest) error {
if len(req.Data) > 0 {
return nil // Already have binary data from file upload
}

if req.TextData == "" {
return nil // No text data either (validation should have caught this)
}

// Check if TextData is a URL that we should fetch content from
if IsURL(req.TextData) {
s.logger.Info("TextData is a URL, fetching content",
"url", req.TextData,
"workspace_id", req.WorkspaceID)

data, err := s.urlFetcher.Fetch(ctx, req.TextData)
if err != nil {
return fmt.Errorf("failed to fetch URL %q: %w", req.TextData, err)
}

req.Data = data
if req.Name == "" {
req.Name = req.TextData
}
s.logger.Info("URL content fetched successfully",
"url", req.TextData,
"data_size", len(data))
return nil
}

// TextData is raw text (curl command, JSON, YAML, etc.)
req.Data = []byte(req.TextData)
s.logger.Debug("Converted text data to binary",
"workspace_id", req.WorkspaceID,
"data_size", len(req.Data))

return nil
}

// DetectFormat performs format detection on the provided data
func (s *Service) DetectFormat(ctx context.Context, data []byte) (*DetectionResult, error) {
if len(data) == 0 {
Expand Down
52 changes: 52 additions & 0 deletions packages/server/internal/api/rimportv2/translators.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/the-dev-tools/dev-tools/packages/server/pkg/service/shttp"
"github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/harv2"
"github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/tcurlv2"
"github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/topenapiv2"
"github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/tpostmanv2"
yamlflowsimplev2 "github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/yamlflowsimplev2"
)
Expand Down Expand Up @@ -90,6 +91,7 @@ func NewTranslatorRegistry(httpService *shttp.HTTPService) *TranslatorRegistry {
registry.RegisterTranslator(NewYAMLTranslator())
registry.RegisterTranslator(NewCURLTranslator())
registry.RegisterTranslator(NewPostmanTranslator())
registry.RegisterTranslator(NewOpenAPITranslator())
registry.RegisterTranslator(NewJSONTranslator())

return registry
Expand Down Expand Up @@ -407,6 +409,56 @@ func (t *PostmanTranslator) Translate(ctx context.Context, data []byte, workspac
return result, nil
}

// OpenAPITranslator implements Translator for OpenAPI/Swagger spec format
type OpenAPITranslator struct {
detector *FormatDetector
}

// NewOpenAPITranslator creates a new OpenAPI translator
func NewOpenAPITranslator() *OpenAPITranslator {
return &OpenAPITranslator{
detector: NewFormatDetector(),
}
}

func (t *OpenAPITranslator) GetFormat() Format {
return FormatOpenAPI
}

func (t *OpenAPITranslator) Validate(data []byte) error {
return t.detector.ValidateFormat(data, FormatOpenAPI)
}

func (t *OpenAPITranslator) Translate(ctx context.Context, data []byte, workspaceID idwrap.IDWrap) (*TranslationResult, error) {
opts := topenapiv2.ConvertOptions{
WorkspaceID: workspaceID,
}

resolved, err := topenapiv2.ConvertOpenAPI(data, opts)
if err != nil {
return nil, fmt.Errorf("failed to convert OpenAPI spec: %w", err)
}

result := &TranslationResult{
HTTPRequests: resolved.HTTPRequests,
Files: resolved.Files,
Headers: resolved.Headers,
SearchParams: resolved.SearchParams,
BodyRaw: resolved.BodyRaw,
Asserts: resolved.Asserts,
Flows: []mflow.Flow{resolved.Flow},
Nodes: resolved.Nodes,
RequestNodes: resolved.RequestNodes,
Edges: resolved.Edges,
ProcessedAt: time.Now().UnixMilli(),
}

// Extract domains from HTTP requests
result.Domains = extractDomainsFromHTTP(result.HTTPRequests)

return result, nil
}

// JSONTranslator implements Translator for generic JSON format
type JSONTranslator struct {
detector *FormatDetector
Expand Down
Loading