httpclient

Documentation

Overview

Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• Constants
• type BasicConfig
• type HttpClientConfigBean
• type HttpClientOrigin
  • func (h *HttpClientOrigin) Init(stageContext api.StageContext) []validation.Issue
  • func (h *HttpClientOrigin) Produce(lastSourceOffset *string, maxBatchSize int, batchMaker api.BatchMaker) (*string, error)
• type HttpStatusResponseActionConfigBean
• type HttpTimeoutResponseActionConfigBean
• type PaginationConfigBean

Constants

const (
	Library         = "streamsets-datacollector-basic-lib"
	StageName       = "com_streamsets_pipeline_stage_origin_http_HttpClientDSource"
	Polling         = "POLLING"
	Streaming       = "STREAMING"
	Batch           = "BATCH"
	HTTP03ErrorCode = "Error fetching resource. Status Code: %s, Reason: %s"
	HTTP32ErrorCode = "Error executing request: %s"
)

Types

type BasicConfig

type BasicConfig struct {
	MaxBatchSize float64 `ConfigDef:"type=NUMBER,required=true"`
	MaxWaitTime  float64 `ConfigDef:"type=NUMBER,required=true"`
}

type HttpClientConfigBean

type HttpClientConfigBean struct {
	ResourceUrl                 string                               `ConfigDef:"type=STRING,required=true"`
	Headers                     map[string]string                    `ConfigDef:"type=MAP,required=true"`
	HttpMethod                  string                               `ConfigDef:"type=STRING,required=true"`
	TimeZoneID                  string                               `ConfigDef:"type=STRING,required=true"`
	RequestBody                 string                               `ConfigDef:"type=STRING,required=true"`
	DefaultRequestContentType   string                               `ConfigDef:"type=STRING,required=true"`
	HttpMode                    string                               `ConfigDef:"type=STRING,required=true"`
	PollingInterval             float64                              `ConfigDef:"type=NUMBER,required=true"`
	Basic                       BasicConfig                          `ConfigDefBean:"name=basic"`
	Client                      httpcommon.ClientConfigBean          `ConfigDefBean:"client"`
	DataFormat                  string                               `ConfigDef:"type=STRING,required=true"`
	DataFormatConfig            dataparser.DataParserFormatConfig    `ConfigDefBean:"dataFormatConfig"`
	ResponseStatusActionConfigs []HttpStatusResponseActionConfigBean `ConfigDef:"type=MODEL,evaluation=EXPLICIT" ListBeanModel:"name=responseStatusActionConfigs"`
	ResponseTimeoutActionConfig HttpTimeoutResponseActionConfigBean  `ConfigDefBean:"responseTimeoutActionConfig"`
	Pagination                  PaginationConfigBean                 `ConfigDefBean:"pagination"`
}

type HttpClientOrigin

type HttpClientOrigin struct {
	*common.BaseStage
	*httpcommon.HttpCommon
	Conf HttpClientConfigBean `ConfigDefBean:"name=conf"`
}

func (*HttpClientOrigin) Init

func (h *HttpClientOrigin) Init(stageContext api.StageContext) []validation.Issue

func (*HttpClientOrigin) Produce

func (h *HttpClientOrigin) Produce(
	lastSourceOffset *string,
	maxBatchSize int,
	batchMaker api.BatchMaker,
) (*string, error)

type HttpStatusResponseActionConfigBean

type HttpStatusResponseActionConfigBean struct {
	StatusCode      float64 `ConfigDef:"type=NUMBER,required=true"`
	Action          string  `ConfigDef:"type=STRING,required=true"`
	BackoffInterval float64 `ConfigDef:"type=NUMBER,required=true"`
	MaxNumRetries   float64 `ConfigDef:"type=NUMBER,required=true"`
}

type HttpTimeoutResponseActionConfigBean

type HttpTimeoutResponseActionConfigBean struct {
	Action          string  `ConfigDef:"type=STRING,required=true"`
	BackoffInterval float64 `ConfigDef:"type=NUMBER,required=true"`
	MaxNumRetries   float64 `ConfigDef:"type=NUMBER,required=true"`
}

type PaginationConfigBean

type PaginationConfigBean struct {
	Mode              string  `ConfigDef:"type=STRING,required=true"`
	NextPageFieldPath string  `ConfigDef:"type=STRING,required=true"`
	StopCondition     string  `ConfigDef:"type=STRING,required=true"`
	StartAt           float64 `ConfigDef:"type=NUMBER,required=true"`
	ResultFieldPath   string  `ConfigDef:"type=STRING,required=true"`
	KeepAllFields     bool    `ConfigDef:"type=BOOLEAN,required=true"`
	RateLimit         float64 `ConfigDef:"type=NUMBER,required=true"`
}