spark_job_definition

Creates, updates, deletes, gets or lists a spark_job_definition resource.

Overview

Name	spark_job_definition
Type	Resource
Id	azure.synapse_artifacts.spark_job_definition

Fields

The following fields are returned by SELECT queries:

get_spark_job_definition

Name	Datatype	Description
id	string	Fully qualified resource ID for the resource. Ex - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. # pylint: disable=line-too-long
name	string	The name of the resource.
	object	Unmatched properties from the message are deserialized to this collection.
description	string	The description of the Spark job definition.
etag	string	Resource Etag.
folder	object	The folder that this Spark job definition is in. If not specified, this Spark job definition will appear at the root level.
jobProperties	object	The properties of the Spark job. Required.
language	string	The language of the Spark application.
requiredSparkVersion	string	The required Spark version of the application.
targetBigDataPool	object	Big data pool reference. Required.
targetSparkConfiguration	object	The spark configuration of the spark job.
type	string	The type of the resource. E.g. "Microsoft.Compute/virtualMachines" or "Microsoft.Storage/storageAccounts".

get_spark_job_definitions_by_workspace

Name	Datatype	Description
id	string	Fully qualified resource ID for the resource. Ex - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. # pylint: disable=line-too-long
name	string	The name of the resource.
	object	Unmatched properties from the message are deserialized to this collection.
description	string	The description of the Spark job definition.
etag	string	Resource Etag.
folder	object	The folder that this Spark job definition is in. If not specified, this Spark job definition will appear at the root level.
jobProperties	object	The properties of the Spark job. Required.
language	string	The language of the Spark application.
requiredSparkVersion	string	The required Spark version of the application.
targetBigDataPool	object	Big data pool reference. Required.
targetSparkConfiguration	object	The spark configuration of the spark job.
type	string	The type of the resource. E.g. "Microsoft.Compute/virtualMachines" or "Microsoft.Storage/storageAccounts".

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get_spark_job_definition	select	spark_job_definition_name, endpoint	If-None-Match	Gets a Spark Job Definition.
get_spark_job_definitions_by_workspace	select	endpoint		Lists spark job definitions.
create_or_update_spark_job_definition	insert	spark_job_definition_name, endpoint, targetBigDataPool, jobProperties	If-Match	Creates or updates a Spark Job Definition.
create_or_update_spark_job_definition	replace	spark_job_definition_name, endpoint, targetBigDataPool, jobProperties	If-Match	Creates or updates a Spark Job Definition.
delete_spark_job_definition	delete	spark_job_definition_name, endpoint		Deletes a Spark Job Definition.
execute_spark_job_definition	exec	spark_job_definition_name, endpoint		Executes the spark job definition.
rename_spark_job_definition	exec	spark_job_definition_name, endpoint		Renames a sparkJobDefinition.
debug_spark_job_definition	exec	endpoint, targetBigDataPool, jobProperties		Debug the spark job definition.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
endpoint	string	The service endpoint host (no scheme). (default: )
spark_job_definition_name	string	The spark job definition name. Required.
If-Match	string	ETag of the Spark Job Definition entity. Should only be specified for update, for which it should match existing entity or can be * for unconditional update. Default value is None.
If-None-Match	string	ETag of the Spark Job Definition entity. Should only be specified for get. If the ETag matches the existing entity tag, or if * was provided, then no content will be returned. Default value is None.

SELECT examples

get_spark_job_definition

Gets a Spark Job Definition.

SELECT
id,
name,
,
description,
etag,
folder,
jobProperties,
language,
requiredSparkVersion,
targetBigDataPool,
targetSparkConfiguration,
type
FROM azure.synapse_artifacts.spark_job_definition
WHERE spark_job_definition_name = '{{ spark_job_definition_name }}' -- required
AND endpoint = '{{ endpoint }}' -- required
AND If-None-Match = '{{ If-None-Match }}'
;

get_spark_job_definitions_by_workspace

Lists spark job definitions.

SELECT
id,
name,
,
description,
etag,
folder,
jobProperties,
language,
requiredSparkVersion,
targetBigDataPool,
targetSparkConfiguration,
type
FROM azure.synapse_artifacts.spark_job_definition
WHERE endpoint = '{{ endpoint }}' -- required
;

INSERT examples

create_or_update_spark_job_definition

Creates or updates a Spark Job Definition.

INSERT INTO azure.synapse_artifacts.spark_job_definition (
,
description,
targetBigDataPool,
targetSparkConfiguration,
requiredSparkVersion,
language,
jobProperties,
folder,
spark_job_definition_name,
endpoint,
If-Match
)
SELECT 
'{{  }}',
'{{ description }}',
'{{ targetBigDataPool }}' /* required */,
'{{ targetSparkConfiguration }}',
'{{ requiredSparkVersion }}',
'{{ language }}',
'{{ jobProperties }}' /* required */,
'{{ folder }}',
'{{ spark_job_definition_name }}',
'{{ endpoint }}',
'{{ If-Match }}'
RETURNING
id,
name,
etag,
properties,
type
;

Manifest

# Description fields are for documentation purposes
- name: spark_job_definition
props:
  - name: spark_job_definition_name
    value: "{{ spark_job_definition_name }}"
    description: Required parameter for the spark_job_definition resource.
  - name: endpoint
    value: "{{ endpoint }}"
    description: Required parameter for the spark_job_definition resource.
  - name: 
    value: "{{  }}"
    description: |
      Unmatched properties from the message are deserialized to this collection.
  - name: description
    value: "{{ description }}"
    description: |
      The description of the Spark job definition.
  - name: targetBigDataPool
    description: |
      Big data pool reference. Required.
    value:
      type: "{{ type }}"
      referenceName: "{{ referenceName }}"
  - name: targetSparkConfiguration
    description: |
      The spark configuration of the spark job.
    value:
      type: "{{ type }}"
      referenceName: "{{ referenceName }}"
  - name: requiredSparkVersion
    value: "{{ requiredSparkVersion }}"
    description: |
      The required Spark version of the application.
  - name: language
    value: "{{ language }}"
    description: |
      The language of the Spark application.
  - name: jobProperties
    description: |
      The properties of the Spark job. Required.
    value:
      : "{{  }}"
      name: "{{ name }}"
      file: "{{ file }}"
      className: "{{ className }}"
      conf: "{{ conf }}"
      args:
        - "{{ args }}"
      jars:
        - "{{ jars }}"
      files:
        - "{{ files }}"
      archives:
        - "{{ archives }}"
      driverMemory: "{{ driverMemory }}"
      driverCores: {{ driverCores }}
      executorMemory: "{{ executorMemory }}"
      executorCores: {{ executorCores }}
      numExecutors: {{ numExecutors }}
  - name: folder
    description: |
      The folder that this Spark job definition is in. If not specified, this Spark job definition will appear at the root level.
    value:
      name: "{{ name }}"
  - name: If-Match
    value: "{{ If-Match }}"
    description: ETag of the Spark Job Definition entity. Should only be specified for update, for which it should match existing entity or can be * for unconditional update. Default value is None.
    description: ETag of the Spark Job Definition entity. Should only be specified for update, for which it should match existing entity or can be * for unconditional update. Default value is None.

REPLACE examples

create_or_update_spark_job_definition

Creates or updates a Spark Job Definition.

REPLACE azure.synapse_artifacts.spark_job_definition
SET 
 = '{{  }}',
description = '{{ description }}',
targetBigDataPool = '{{ targetBigDataPool }}',
targetSparkConfiguration = '{{ targetSparkConfiguration }}',
requiredSparkVersion = '{{ requiredSparkVersion }}',
language = '{{ language }}',
jobProperties = '{{ jobProperties }}',
folder = '{{ folder }}'
WHERE 
spark_job_definition_name = '{{ spark_job_definition_name }}' --required
AND endpoint = '{{ endpoint }}' --required
AND targetBigDataPool = '{{ targetBigDataPool }}' --required
AND jobProperties = '{{ jobProperties }}' --required
AND If-Match = '{{ If-Match}}'
RETURNING
id,
name,
etag,
properties,
type;

DELETE examples

delete_spark_job_definition

Deletes a Spark Job Definition.

DELETE FROM azure.synapse_artifacts.spark_job_definition
WHERE spark_job_definition_name = '{{ spark_job_definition_name }}' --required
AND endpoint = '{{ endpoint }}' --required
;

Lifecycle Methods

execute_spark_job_definition

Executes the spark job definition.

EXEC azure.synapse_artifacts.spark_job_definition.execute_spark_job_definition 
@spark_job_definition_name='{{ spark_job_definition_name }}' --required, 
@endpoint='{{ endpoint }}' --required
;

rename_spark_job_definition

Renames a sparkJobDefinition.

EXEC azure.synapse_artifacts.spark_job_definition.rename_spark_job_definition 
@spark_job_definition_name='{{ spark_job_definition_name }}' --required, 
@endpoint='{{ endpoint }}' --required
;

debug_spark_job_definition

Debug the spark job definition.

EXEC azure.synapse_artifacts.spark_job_definition.debug_spark_job_definition 
@endpoint='{{ endpoint }}' --required 
@@json=
'{
"": "{{  }}", 
"description": "{{ description }}", 
"targetBigDataPool": "{{ targetBigDataPool }}", 
"targetSparkConfiguration": "{{ targetSparkConfiguration }}", 
"requiredSparkVersion": "{{ requiredSparkVersion }}", 
"language": "{{ language }}", 
"jobProperties": "{{ jobProperties }}", 
"folder": "{{ folder }}"
}'
;