pipelines

Creates, updates, deletes, gets or lists a pipelines resource.

Overview

Name	pipelines
Type	Resource
Id	azure.data_factory.pipelines

Fields

The following fields are returned by SELECT queries:

get

Name	Datatype	Description
id	string	Fully qualified resource ID for the resource. Ex - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}.
name	string	The name of the resource.
activities	array	List of activities in pipeline.
annotations	array	List of tags that can be used for describing the Pipeline.
concurrency	integer	The max number of concurrent runs for the pipeline.
description	string	The description of the pipeline.
etag	string	"If etag is provided in the response body, it may also be provided as a header per the normal etag convention. Entity tags are used for comparing two or more entities from the same requested resource. HTTP/1.1 uses entity tags in the etag (section 14.19), If-Match (section 14.24), If-None-Match (section 14.26), and If-Range (section 14.27) header fields.").
folder	object	The folder that this Pipeline is in. If not specified, Pipeline will appear at the root level.
parameters	object	List of parameters for pipeline.
policy	object	Pipeline Policy.
runDimensions	object	Dimensions emitted by Pipeline.
systemData	object	Azure Resource Manager metadata containing createdBy and modifiedBy information.
type	string	The type of the resource. E.g. "Microsoft.Compute/virtualMachines" or "Microsoft.Storage/storageAccounts".
variables	object	List of variables for pipeline.

list_by_factory

Name	Datatype	Description
id	string	Fully qualified resource ID for the resource. Ex - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}.
name	string	The name of the resource.
activities	array	List of activities in pipeline.
annotations	array	List of tags that can be used for describing the Pipeline.
concurrency	integer	The max number of concurrent runs for the pipeline.
description	string	The description of the pipeline.
etag	string	"If etag is provided in the response body, it may also be provided as a header per the normal etag convention. Entity tags are used for comparing two or more entities from the same requested resource. HTTP/1.1 uses entity tags in the etag (section 14.19), If-Match (section 14.24), If-None-Match (section 14.26), and If-Range (section 14.27) header fields.").
folder	object	The folder that this Pipeline is in. If not specified, Pipeline will appear at the root level.
parameters	object	List of parameters for pipeline.
policy	object	Pipeline Policy.
runDimensions	object	Dimensions emitted by Pipeline.
systemData	object	Azure Resource Manager metadata containing createdBy and modifiedBy information.
type	string	The type of the resource. E.g. "Microsoft.Compute/virtualMachines" or "Microsoft.Storage/storageAccounts".
variables	object	List of variables for pipeline.

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get	select	resource_group_name, factory_name, pipeline_name, subscription_id		Gets a pipeline.
list_by_factory	select	resource_group_name, factory_name, subscription_id		Lists pipelines.
create_or_update	insert	resource_group_name, factory_name, pipeline_name, subscription_id, properties		Creates or updates a pipeline.
create_or_update	replace	resource_group_name, factory_name, pipeline_name, subscription_id, properties		Creates or updates a pipeline.
delete	delete	resource_group_name, factory_name, pipeline_name, subscription_id		Deletes a pipeline.
create_run	exec	resource_group_name, factory_name, pipeline_name, subscription_id	referencePipelineRunId, isRecovery, startActivityName, startFromFailure	Creates a run of a pipeline.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
factory_name	string	The factory name. Required.
pipeline_name	string	The pipeline name. Required.
resource_group_name	string	The name of the resource group. The name is case insensitive. Required.
subscription_id	string
isRecovery	boolean	Recovery mode flag. If recovery mode is set to true, the specified referenced pipeline run and the new run will be grouped under the same groupId. Default value is None.
referencePipelineRunId	string	The pipeline run identifier. If run ID is specified the parameters of the specified run will be used to create a new run. Default value is None.
startActivityName	string	In recovery mode, the rerun will start from this activity. If not specified, all activities will run. Default value is None.
startFromFailure	boolean	In recovery mode, if set to true, the rerun will start from failed activities. The property will be used only if startActivityName is not specified. Default value is None.

SELECT examples

get

Gets a pipeline.

SELECT
id,
name,
activities,
annotations,
concurrency,
description,
etag,
folder,
parameters,
policy,
runDimensions,
systemData,
type,
variables
FROM azure.data_factory.pipelines
WHERE resource_group_name = '{{ resource_group_name }}' -- required
AND factory_name = '{{ factory_name }}' -- required
AND pipeline_name = '{{ pipeline_name }}' -- required
AND subscription_id = '{{ subscription_id }}' -- required
;

list_by_factory

Lists pipelines.

SELECT
id,
name,
activities,
annotations,
concurrency,
description,
etag,
folder,
parameters,
policy,
runDimensions,
systemData,
type,
variables
FROM azure.data_factory.pipelines
WHERE resource_group_name = '{{ resource_group_name }}' -- required
AND factory_name = '{{ factory_name }}' -- required
AND subscription_id = '{{ subscription_id }}' -- required
;

INSERT examples

create_or_update

Creates or updates a pipeline.

INSERT INTO azure.data_factory.pipelines (
properties,
resource_group_name,
factory_name,
pipeline_name,
subscription_id
)
SELECT 
'{{ properties }}' /* required */,
'{{ resource_group_name }}',
'{{ factory_name }}',
'{{ pipeline_name }}',
'{{ subscription_id }}'
RETURNING
id,
name,
etag,
properties,
systemData,
type
;

Manifest

# Description fields are for documentation purposes
- name: pipelines
props:
  - name: resource_group_name
    value: "{{ resource_group_name }}"
    description: Required parameter for the pipelines resource.
  - name: factory_name
    value: "{{ factory_name }}"
    description: Required parameter for the pipelines resource.
  - name: pipeline_name
    value: "{{ pipeline_name }}"
    description: Required parameter for the pipelines resource.
  - name: subscription_id
    value: "{{ subscription_id }}"
    description: Required parameter for the pipelines resource.
  - name: properties
    description: |
      Properties of the pipeline. Required.
    value:
      description: "{{ description }}"
      activities:
        - name: "{{ name }}"
          type: "{{ type }}"
          description: "{{ description }}"
          state: "{{ state }}"
          onInactiveMarkAs: "{{ onInactiveMarkAs }}"
          dependsOn: "{{ dependsOn }}"
          userProperties: "{{ userProperties }}"
      parameters: "{{ parameters }}"
      variables: "{{ variables }}"
      concurrency: {{ concurrency }}
      annotations: "{{ annotations }}"
      runDimensions: "{{ runDimensions }}"
      folder:
        name: "{{ name }}"
      policy:
        elapsedTimeMetric:
          duration: "{{ duration }}"

REPLACE examples

create_or_update

Creates or updates a pipeline.

REPLACE azure.data_factory.pipelines
SET 
properties = '{{ properties }}'
WHERE 
resource_group_name = '{{ resource_group_name }}' --required
AND factory_name = '{{ factory_name }}' --required
AND pipeline_name = '{{ pipeline_name }}' --required
AND subscription_id = '{{ subscription_id }}' --required
AND properties = '{{ properties }}' --required
RETURNING
id,
name,
etag,
properties,
systemData,
type;

DELETE examples

delete

Deletes a pipeline.

DELETE FROM azure.data_factory.pipelines
WHERE resource_group_name = '{{ resource_group_name }}' --required
AND factory_name = '{{ factory_name }}' --required
AND pipeline_name = '{{ pipeline_name }}' --required
AND subscription_id = '{{ subscription_id }}' --required
;

Lifecycle Methods

create_run

Creates a run of a pipeline.

EXEC azure.data_factory.pipelines.create_run 
@resource_group_name='{{ resource_group_name }}' --required, 
@factory_name='{{ factory_name }}' --required, 
@pipeline_name='{{ pipeline_name }}' --required, 
@subscription_id='{{ subscription_id }}' --required, 
@referencePipelineRunId='{{ referencePipelineRunId }}', 
@isRecovery={{ isRecovery }}, 
@startActivityName='{{ startActivityName }}', 
@startFromFailure={{ startFromFailure }}
;