Go Integration Guide

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"time"
)

const baseURL = "https://api.crawlora.net/api/v1"

type SearchRequest struct {
	Country  string `json:"country,omitempty"`
	Keyword  string `json:"keyword"`
	Language string `json:"language,omitempty"`
	Limit    int    `json:"limit,omitempty"`
	Page     int    `json:"page,omitempty"`
}

type SearchResponse struct {
	Code int    `json:"code"`
	Msg  string `json:"msg"`
	Data struct {
		Result []struct {
			Position int    `json:"position,omitempty"`
			Title    string `json:"title,omitempty"`
			Link     string `json:"link,omitempty"`
			Snippet  string `json:"Snippet,omitempty"`
		} `json:"result,omitempty"`
	} `json:"data,omitempty"`
}

func crawloraRequest(ctx context.Context, client *http.Client, path string, payload any, out any) error {
	apiKey := os.Getenv("CRAWLORA_API_KEY")
	if apiKey == "" {
		return fmt.Errorf("missing CRAWLORA_API_KEY")
	}

	body, err := json.Marshal(payload)
	if err != nil {
		return err
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+path, bytes.NewReader(body))
	if err != nil {
		return err
	}

	req.Header.Set("x-api-key", apiKey)
	req.Header.Set("Content-Type", "application/json")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	respBody, err := io.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return fmt.Errorf("crawlora request failed: status=%d body=%s", resp.StatusCode, string(respBody))
	}

	return json.Unmarshal(respBody, out)
}

func main() {
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	defer cancel()

	client := &http.Client{Timeout: 65 * time.Second}

	var result SearchResponse
	err := crawloraRequest(ctx, client, "/google/search", SearchRequest{
		Country:  "us",
		Keyword:  "best CRM software",
		Language: "en",
		Limit:    10,
		Page:     1,
	}, &result)
	if err != nil {
		panic(err)
	}

	fmt.Printf("%+v\n", result.Data.Result)
}

type Job struct {
	Keyword string
}

func worker(ctx context.Context, client *http.Client, jobs <-chan Job, results chan<- SearchResponse) {
	for {
		select {
		case <-ctx.Done():
			return
		case job, ok := <-jobs:
			if !ok {
				return
			}

			var out SearchResponse
			err := crawloraRequest(ctx, client, "/google/search", SearchRequest{
				Keyword:  job.Keyword,
				Country:  "us",
				Language: "en",
				Limit:    10,
				Page:     1,
			}, &out)
			if err == nil {
				results <- out
			}
		}
	}
}