spark_session

Creates, updates, deletes, gets or lists a spark_session resource.

Overview

Name	spark_session
Type	Resource
Id	azure.synapse_spark.spark_session

Fields

The following fields are returned by SELECT queries:

get_spark_statement

Name	Datatype	Description
id	integer
code	string
output	object	SparkStatementOutput. All required parameters must be populated in order to send to Azure.
state	string

get_spark_session

Name	Datatype	Description
id	integer
name	string
appId	string
appInfo	object
artifactId	string
errorInfo	array
jobType	string
livyInfo	object	SparkSessionState.
log	array
pluginInfo	object	SparkServicePlugin.
result	string
schedulerInfo	object	SparkScheduler.
sparkPoolName	string
state	string
submitterId	string
submitterName	string
tags	object
workspaceName	string

get_spark_sessions

Name	Datatype	Description
from	integer
sessions	array
total	integer

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get_spark_statement	select	session_id, statement_id, livy_api_version, spark_pool_name, endpoint		Gets a single statement within a spark session.
get_spark_session	select	session_id, livy_api_version, spark_pool_name, endpoint	detailed	Gets a single spark session.
get_spark_sessions	select	livy_api_version, spark_pool_name, endpoint	from, size, detailed	List all spark sessions which are running under a particular spark pool.
create_spark_session	insert	livy_api_version, spark_pool_name, endpoint, name	detailed	Create new spark session.
cancel_spark_session	delete	session_id, livy_api_version, spark_pool_name, endpoint		Cancels a running spark session.
get_spark_statements	exec	session_id, livy_api_version, spark_pool_name, endpoint		Gets a list of statements within a spark session.
create_spark_statement	exec	session_id, livy_api_version, spark_pool_name, endpoint		Create statement within a spark session.
reset_spark_session_timeout	exec	session_id, livy_api_version, spark_pool_name, endpoint		Sends a keep alive call to the current session to reset the session timeout.
cancel_spark_statement	exec	session_id, statement_id, livy_api_version, spark_pool_name, endpoint		Kill a statement within a session.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
endpoint	string	The service endpoint host (no scheme). (default: )
livy_api_version	string
session_id	integer	Identifier for the session.
spark_pool_name	string
statement_id	integer	Identifier for the statement.
detailed	boolean	Optional query param specifying whether detailed response is returned beyond plain livy.
from	integer	Optional param specifying which index the list should begin from.
size	integer	Optional param specifying the size of the returned list. By default it is 20 and that is the maximum.

SELECT examples

get_spark_statement

Gets a single statement within a spark session.

SELECT
id,
code,
output,
state
FROM azure.synapse_spark.spark_session
WHERE session_id = '{{ session_id }}' -- required
AND statement_id = '{{ statement_id }}' -- required
AND livy_api_version = '{{ livy_api_version }}' -- required
AND spark_pool_name = '{{ spark_pool_name }}' -- required
AND endpoint = '{{ endpoint }}' -- required
;

get_spark_session

Gets a single spark session.

SELECT
id,
name,
appId,
appInfo,
artifactId,
errorInfo,
jobType,
livyInfo,
log,
pluginInfo,
result,
schedulerInfo,
sparkPoolName,
state,
submitterId,
submitterName,
tags,
workspaceName
FROM azure.synapse_spark.spark_session
WHERE session_id = '{{ session_id }}' -- required
AND livy_api_version = '{{ livy_api_version }}' -- required
AND spark_pool_name = '{{ spark_pool_name }}' -- required
AND endpoint = '{{ endpoint }}' -- required
AND detailed = '{{ detailed }}'
;

get_spark_sessions

List all spark sessions which are running under a particular spark pool.

SELECT
from,
sessions,
total
FROM azure.synapse_spark.spark_session
WHERE livy_api_version = '{{ livy_api_version }}' -- required
AND spark_pool_name = '{{ spark_pool_name }}' -- required
AND endpoint = '{{ endpoint }}' -- required
AND from = '{{ from }}'
AND size = '{{ size }}'
AND detailed = '{{ detailed }}'
;

INSERT examples

create_spark_session

Create new spark session.

INSERT INTO azure.synapse_spark.spark_session (
tags,
artifactId,
name,
file,
className,
args,
jars,
pyFiles,
files,
archives,
conf,
driverMemory,
driverCores,
executorMemory,
executorCores,
numExecutors,
livy_api_version,
spark_pool_name,
endpoint,
detailed
)
SELECT 
'{{ tags }}',
'{{ artifactId }}',
'{{ name }}' /* required */,
'{{ file }}',
'{{ className }}',
'{{ args }}',
'{{ jars }}',
'{{ pyFiles }}',
'{{ files }}',
'{{ archives }}',
'{{ conf }}',
'{{ driverMemory }}',
{{ driverCores }},
'{{ executorMemory }}',
{{ executorCores }},
{{ numExecutors }},
'{{ livy_api_version }}',
'{{ spark_pool_name }}',
'{{ endpoint }}',
'{{ detailed }}'
RETURNING
id,
name,
appId,
appInfo,
artifactId,
errorInfo,
jobType,
livyInfo,
log,
pluginInfo,
result,
schedulerInfo,
sparkPoolName,
state,
submitterId,
submitterName,
tags,
workspaceName
;

Manifest

# Description fields are for documentation purposes
- name: spark_session
props:
  - name: livy_api_version
    value: "{{ livy_api_version }}"
    description: Required parameter for the spark_session resource.
  - name: spark_pool_name
    value: "{{ spark_pool_name }}"
    description: Required parameter for the spark_session resource.
  - name: endpoint
    value: "{{ endpoint }}"
    description: Required parameter for the spark_session resource.
  - name: tags
    value: "{{ tags }}"
  - name: artifactId
    value: "{{ artifactId }}"
  - name: name
    value: "{{ name }}"
  - name: file
    value: "{{ file }}"
  - name: className
    value: "{{ className }}"
  - name: args
    value:
      - "{{ args }}"
  - name: jars
    value:
      - "{{ jars }}"
  - name: pyFiles
    value:
      - "{{ pyFiles }}"
  - name: files
    value:
      - "{{ files }}"
  - name: archives
    value:
      - "{{ archives }}"
  - name: conf
    value: "{{ conf }}"
  - name: driverMemory
    value: "{{ driverMemory }}"
  - name: driverCores
    value: {{ driverCores }}
  - name: executorMemory
    value: "{{ executorMemory }}"
  - name: executorCores
    value: {{ executorCores }}
  - name: numExecutors
    value: {{ numExecutors }}
  - name: detailed
    value: {{ detailed }}
    description: Optional query param specifying whether detailed response is returned beyond plain livy.
    description: Optional query param specifying whether detailed response is returned beyond plain livy.

DELETE examples

cancel_spark_session

Cancels a running spark session.

DELETE FROM azure.synapse_spark.spark_session
WHERE session_id = '{{ session_id }}' --required
AND livy_api_version = '{{ livy_api_version }}' --required
AND spark_pool_name = '{{ spark_pool_name }}' --required
AND endpoint = '{{ endpoint }}' --required
;

Lifecycle Methods

get_spark_statements

Gets a list of statements within a spark session.

EXEC azure.synapse_spark.spark_session.get_spark_statements 
@session_id='{{ session_id }}' --required, 
@livy_api_version='{{ livy_api_version }}' --required, 
@spark_pool_name='{{ spark_pool_name }}' --required, 
@endpoint='{{ endpoint }}' --required
;

create_spark_statement

Create statement within a spark session.

EXEC azure.synapse_spark.spark_session.create_spark_statement 
@session_id='{{ session_id }}' --required, 
@livy_api_version='{{ livy_api_version }}' --required, 
@spark_pool_name='{{ spark_pool_name }}' --required, 
@endpoint='{{ endpoint }}' --required 
@@json=
'{
"code": "{{ code }}", 
"kind": "{{ kind }}"
}'
;

reset_spark_session_timeout

Sends a keep alive call to the current session to reset the session timeout.

EXEC azure.synapse_spark.spark_session.reset_spark_session_timeout 
@session_id='{{ session_id }}' --required, 
@livy_api_version='{{ livy_api_version }}' --required, 
@spark_pool_name='{{ spark_pool_name }}' --required, 
@endpoint='{{ endpoint }}' --required
;

cancel_spark_statement

Kill a statement within a session.

EXEC azure.synapse_spark.spark_session.cancel_spark_statement 
@session_id='{{ session_id }}' --required, 
@statement_id='{{ statement_id }}' --required, 
@livy_api_version='{{ livy_api_version }}' --required, 
@spark_pool_name='{{ spark_pool_name }}' --required, 
@endpoint='{{ endpoint }}' --required
;