From 412bbe0b37b9c9101fe0c21c09d09b9c542f5669 Mon Sep 17 00:00:00 2001 From: Julien Dessaux Date: Tue, 20 May 2025 21:45:08 +0200 Subject: [PATCH] feat(client): add pagination support Closes #11 --- internal/client/client.go | 60 +++++++++++++++++++++++++++----- internal/client/organizations.go | 10 ++---- internal/client/repositories.go | 34 +++++++++++------- internal/client/teams.go | 10 ++---- internal/client/users.go | 34 +++++++++++------- 5 files changed, 98 insertions(+), 50 deletions(-) diff --git a/internal/client/client.go b/internal/client/client.go index e16eef0..b9883f5 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "strconv" "time" ) @@ -17,6 +18,9 @@ type Client struct { httpClient *http.Client } +const maxItemsPerPage = 50 +const maxItemsPerPageStr = "50" + func NewClient(baseURL *url.URL, apiToken string) *Client { return &Client{ baseURI: baseURL, @@ -31,13 +35,49 @@ func NewClient(baseURL *url.URL, apiToken string) *Client { } } -func (c *Client) Send(ctx context.Context, method string, uriRef *url.URL, payload any, response any) error { +func (c *Client) sendPaginated(ctx context.Context, method string, uriRef *url.URL, payload any, response any) error { + query, err := url.ParseQuery(uriRef.RawQuery) + if err != nil { + return fmt.Errorf("failed to parse query string: %w", err) + } + query.Set("limit", maxItemsPerPageStr) + page := 1 + var rawResponses []json.RawMessage + for { + query.Set("page", strconv.Itoa(page)) + uriRef.RawQuery = query.Encode() + var res json.RawMessage + count, err := c.Send(ctx, method, uriRef, payload, &res) + if err != nil { + return fmt.Errorf("failed to send: %w", err) + } + var oneResponse []json.RawMessage + if err := json.Unmarshal(res, &oneResponse); err != nil { + return fmt.Errorf("failed to unmarshal message: %w", err) + } + rawResponses = append(rawResponses, oneResponse...) + if count <= page*maxItemsPerPage { + break + } + page++ + } + responses, err := json.Marshal(rawResponses) + if err != nil { + return fmt.Errorf("failed to marshal raw responses to paginated request: %w", err) + } + if err := json.Unmarshal(responses, &response); err != nil { + return fmt.Errorf("failed to unmarshal paginated request responses: %w", err) + } + return nil +} + +func (c *Client) Send(ctx context.Context, method string, uriRef *url.URL, payload any, response any) (int, error) { uri := c.baseURI.ResolveReference(uriRef) var payloadReader io.Reader if payload != nil { if body, err := json.Marshal(payload); err != nil { - return fmt.Errorf("cannot marshal payload: %w", err) + return 0, fmt.Errorf("cannot marshal payload: %w", err) } else { payloadReader = bytes.NewReader(body) } @@ -45,24 +85,28 @@ func (c *Client) Send(ctx context.Context, method string, uriRef *url.URL, paylo req, err := http.NewRequestWithContext(ctx, method, uri.String(), payloadReader) if err != nil { - return fmt.Errorf("cannot create request: %w", err) + return 0, fmt.Errorf("cannot create request: %w", err) } req.Header = *c.headers resp, err := c.httpClient.Do(req) if err != nil { - return fmt.Errorf("cannot send request: %w", err) + return 0, fmt.Errorf("cannot send request: %w", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("cannot read response body: %w", err) + return 0, fmt.Errorf("cannot read response body: %w", err) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return fmt.Errorf("non 2XX status code received: %d, %q", resp.StatusCode, body) + return 0, fmt.Errorf("non 2XX status code received: %d, %q", resp.StatusCode, body) } if err = json.Unmarshal(body, response); err != nil { - return fmt.Errorf("response body unmarshal failed: %w", err) + return 0, fmt.Errorf("response body unmarshal failed: %w", err) + } + if count, err := strconv.Atoi(resp.Header.Get("x-total-count")); err != nil { + return 0, nil + } else { + return count, nil } - return nil } diff --git a/internal/client/organizations.go b/internal/client/organizations.go index a3acb17..7f061cf 100644 --- a/internal/client/organizations.go +++ b/internal/client/organizations.go @@ -21,14 +21,8 @@ type Organization struct { func (c *Client) OrganizationsList(ctx context.Context) ([]Organization, error) { var response []Organization - query := make(url.Values) - query.Set("limit", "50") - query.Set("page", "1") - uriRef := url.URL{ - Path: "api/v1/orgs", - RawQuery: query.Encode(), - } - if err := c.Send(ctx, "GET", &uriRef, nil, &response); err != nil { + uriRef := url.URL{Path: "api/v1/orgs"} + if err := c.sendPaginated(ctx, "GET", &uriRef, nil, &response); err != nil { return nil, fmt.Errorf("failed to get organizations: %w", err) } return response, nil diff --git a/internal/client/repositories.go b/internal/client/repositories.go index 718f211..f3bb99e 100644 --- a/internal/client/repositories.go +++ b/internal/client/repositories.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/url" + "strconv" "time" ) @@ -104,19 +105,26 @@ func (c *Client) RepositoriesList(ctx context.Context) ([]Repository, error) { Data []Repository `json:"data"` Ok bool `json:"ok"` } - var response Response + uriRef := url.URL{Path: "api/v1/repos/search"} query := make(url.Values) - query.Set("limit", "50") - query.Set("page", "1") - uriRef := url.URL{ - Path: "api/v1/repos/search", - RawQuery: query.Encode(), + query.Set("limit", maxItemsPerPageStr) + page := 1 + var repositories []Repository + var response Response + for { + query.Set("page", strconv.Itoa(page)) + uriRef.RawQuery = query.Encode() + count, err := c.Send(ctx, "GET", &uriRef, nil, &response) + if err != nil { + return nil, fmt.Errorf("failed to search repositories: %w", err) + } + if !response.Ok { + return nil, fmt.Errorf("got a non OK status when searching repositories") + } + repositories = append(repositories, response.Data...) + if count <= page*maxItemsPerPage { + return repositories, nil + } + page++ } - if err := c.Send(ctx, "GET", &uriRef, nil, &response); err != nil { - return nil, fmt.Errorf("failed to search repositories: %w", err) - } - if !response.Ok { - return response.Data, fmt.Errorf("got a non OK status when querying repos/search") - } - return response.Data, nil } diff --git a/internal/client/teams.go b/internal/client/teams.go index 764f41c..9ab5d51 100644 --- a/internal/client/teams.go +++ b/internal/client/teams.go @@ -21,14 +21,8 @@ type Team struct { func (c *Client) TeamsList(ctx context.Context, organizationName string) ([]Team, error) { var response []Team - query := make(url.Values) - query.Set("limit", "50") - query.Set("page", "1") - uriRef := url.URL{ - Path: path.Join("api/v1/orgs", organizationName, "teams"), - RawQuery: query.Encode(), - } - if err := c.Send(ctx, "GET", &uriRef, nil, &response); err != nil { + uriRef := url.URL{Path: path.Join("api/v1/orgs", organizationName, "teams")} + if err := c.sendPaginated(ctx, "GET", &uriRef, nil, &response); err != nil { return nil, fmt.Errorf("failed to list teams of organization %s: %w", organizationName, err) } return response, nil diff --git a/internal/client/users.go b/internal/client/users.go index be2bcaf..186aed4 100644 --- a/internal/client/users.go +++ b/internal/client/users.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/url" + "strconv" "time" ) @@ -38,19 +39,26 @@ func (c *Client) UsersList(ctx context.Context) ([]User, error) { Data []User `json:"data"` Ok bool `json:"ok"` } - var response Response + uriRef := url.URL{Path: "api/v1/users/search"} query := make(url.Values) - query.Set("limit", "50") - query.Set("page", "1") - uriRef := url.URL{ - Path: "api/v1/users/search", - RawQuery: query.Encode(), + query.Set("limit", maxItemsPerPageStr) + page := 1 + var users []User + var response Response + for { + query.Set("page", strconv.Itoa(page)) + uriRef.RawQuery = query.Encode() + count, err := c.Send(ctx, "GET", &uriRef, nil, &response) + if err != nil { + return nil, fmt.Errorf("failed to search users: %w", err) + } + if !response.Ok { + return nil, fmt.Errorf("got a non OK status when searching users") + } + users = append(users, response.Data...) + if count <= page*maxItemsPerPage { + return users, nil + } + page++ } - if err := c.Send(ctx, "GET", &uriRef, nil, &response); err != nil { - return nil, fmt.Errorf("failed to search users: %w", err) - } - if !response.Ok { - return response.Data, fmt.Errorf("got a non OK status when querying users/search") - } - return response.Data, nil }