RACF Security Engineering Handbook

/* Examples */
P.PAYMT1.ATTLS.TLS      /* Prod, Payment app, AT-TLS, TLS handshake  */
T.PAYMT1.ATTLS.TLS      /* Test equivalent                           */
P.ORDMGT.CICS.TLS       /* Prod, Order Mgmt, CICS, TLS               */
P.ORDMGT.MQ.AUTH        /* Prod, Order Mgmt, MQ, client auth         */

/* Ownership: ring owner = application service ID */
RACDCERT ID(PAYMT1P) ADDRING(P.PAYMT1.ATTLS.TLS)
RACDCERT ID(PAYMT1P) CONNECT(LABEL('P.PAYMT1.TLS.2026') +
  RING(P.PAYMT1.ATTLS.TLS) USAGE(PERSONAL) DEFAULT)
RACDCERT CONNECT(CERTAUTH LABEL('INTERNAL ROOT CA') +
  RING(P.PAYMT1.ATTLS.TLS) USAGE(CERTAUTH))

/* Step 1 — Generate CSR on z/OS */
//GENCSRJB JOB (ACCT),'GEN CSR',CLASS=A,MSGCLASS=X
//STEP1    EXEC PGM=IKJEFT01
//SYSTSPRT DD SYSOUT=*
//SYSTSIN  DD *
  RACDCERT ID(PAYMT1P) GENREQ +
    SUBJECTSDN(CN('paymt1.internal.bank.com') +
               OU('Payments') O('First National Bank') C('US')) +
    WITHLABEL('P.PAYMT1.TLS.2026.CSR') +
    KEYUSAGE(HANDSHAKE)
/*

/* Step 2 — Export CSR for submission to internal CA */
  RACDCERT EXPORT(LABEL('P.PAYMT1.TLS.2026.CSR')) +
    ID(PAYMT1P) DSN('PAYMT1P.CERT.CSR') FORMAT(CERTREQ)

/* Step 3 — (Orchestrator submits CSR to CA via ACME/SCEP) */

/* Step 4 — Import signed certificate back */
  RACDCERT ADD('PAYMT1P.CERT.SIGNED') +
    ID(PAYMT1P) WITHLABEL('P.PAYMT1.TLS.2026') TRUST

/* Step 5 — Connect to ring and set as DEFAULT */
  RACDCERT ID(PAYMT1P) CONNECT(LABEL('P.PAYMT1.TLS.2026') +
    RING(P.PAYMT1.ATTLS.TLS) USAGE(PERSONAL) DEFAULT)

  SETROPTS RACLIST(DIGTCERT) REFRESH

/* Nightly inventory job — runs via JES scheduler at 02:00 */
//NIGHTLY  JOB (ACCT),'NIGHTLY INV',CLASS=B,MSGCLASS=X
//SCAN     EXEC PGM=IKJEFT01,DYNAMNBR=20
//SYSTSPRT DD DSN=CERT.INVENTORY.&&DATE,DISP=(NEW,CATLG),
//            SPACE=(CYL,(5,2)),LRECL=255,RECFM=VB
//SYSTSIN  DD *
  RACDCERT CERTAUTH LIST
  RACDCERT SITE LIST
  RACDCERT ID(*) LIST
  RACDCERT ID(*) LISTRING(*)
  RACDCERT SITE LISTRING(*)
/*
//PARSE    EXEC PGM=IKJEFT01
//SYSTSIN  DD *
  /* Orchestrator parses output, upserts inventory DB,  */
  /* computes expiry deltas, flags drift vs prior night  */
/*

# ── Zowe CLI: full AI-to-z/OS automation flow ──────────────────────

# 1. Orchestrator writes AI-generated JCL to a temp dataset
zowe files upload file-to-data-set ./ai_generated.jcl   "PAYMT1P.AI.GENCERT.JCL" --data-set-type SEQ

# 2. Engineer reviews JCL in Zowe Explorer (VS Code) — APPROVAL STEP
#    zowe files download data-set "PAYMT1P.AI.GENCERT.JCL"

# 3. After approval, orchestrator submits the job
zowe jobs submit data-set "PAYMT1P.AI.GENCERT.JCL" --view-all-spool-content

# 4. Retrieve JES output for inventory parsing by AI
zowe jobs list jobs --owner PAYMT1P --prefix GENCRTJB
zowe jobs view spool-file-by-id JOB12345 2 > spool_output.txt

# 5. Pipe spool output to LLM extraction for inventory update
curl -s -X POST $LLM_API_URL   -H "Authorization: Bearer $LLM_API_KEY"   -d "{"prompt": "Extract cert fields from: $(cat spool_output.txt)"}"   | jq '.result' >> inventory_db_upsert.json

# 6. Verify certificate is now in RACF
zowe tso issue command "RACDCERT ID(PAYMT1P) LIST(LABEL('P.PAYMT1.TLS.2026'))"

/* Step 1: Identify the ring and owner from the failing application log */
zowe tso issue command "RACDCERT ID(PAYMT1P) LISTRING(P.PAYMT1.ATTLS.TLS)"
/* Look for: PERSONAL cert connected as DEFAULT, CERTAUTH anchor present */

/* Step 2: Check the certificate itself */
zowe tso issue command "RACDCERT ID(PAYMT1P) LIST(LABEL('P.PAYMT1.TLS.2026'))"
/* Verify: TRUST status = TRUST, NOTAFTER date is in the future */

/* Step 3: If CERTAUTH anchor is missing — add it */
zowe tso issue command   "RACDCERT CONNECT(CERTAUTH LABEL('INTERNAL ROOT CA') +
   RING(P.PAYMT1.ATTLS.TLS) USAGE(CERTAUTH))"

/* Step 4: ALWAYS run refresh after any ring change */
zowe tso issue command "SETROPTS RACLIST(DIGTCERT) REFRESH"

/* Step 5: Verify fix — rerun LISTRING and confirm CERTAUTH entry present */
zowe tso issue command "RACDCERT ID(PAYMT1P) LISTRING(P.PAYMT1.ATTLS.TLS)"

/* ServiceNow: Update the incident work notes with steps taken and outcome */
# curl -X PATCH .../incident/SYS_ID -d '{"work_notes": "Added CERTAUTH anchor. REFRESH run. Verified."}'
# If root cause was a missed REFRESH after a prior change, link to that CR.

/* Step 1: Confirm the label exists for the user */
zowe tso issue command "RACDCERT ID(PAYMT1P) LIST"
/* If label is absent: cert was deleted or never created */

/* Step 2: Check if label exists under a different owner */
zowe tso issue command "RACDCERT CERTAUTH LIST"
zowe tso issue command "RACDCERT SITE LIST"
/* Label may have been created under CERTAUTH or SITE instead of the app ID */

/* Step 3: Check ring for orphaned connections */
zowe tso issue command "RACDCERT ID(PAYMT1P) LISTRING(P.PAYMT1.ATTLS.TLS)"
/* If ring references a label that no longer exists: REMOVE the stale entry */
zowe tso issue command   "RACDCERT ID(PAYMT1P) REMOVE(LABEL('OLD.LABEL') RING(P.PAYMT1.ATTLS.TLS))"

/* Step 4: Re-run the renewal SOP (SOP-2 or SOP-3) to recreate the certificate */

/* ServiceNow: If cert was deleted outside of a change window, raise a Security Incident */
# Attach RACDCERT LIST output as evidence; assign to RACF Security Engineering

/* Step 1: Identify which RACF profile is blocking the operation */
/* ICH408I message includes the profile name and class */
/* Example: ICH408I USER(AUTOBOT) RACDCERT CLASS(DIGTCERT) */

/* Step 2: Check current permits on the automation ID */
zowe tso issue command "RLIST DIGTCERT ** AUTHUSER"
/* Or for a specific profile: */
zowe tso issue command "RLIST DIGTCERT 'PAYMT1P.**' AUTHUSER"

/* Step 3: Grant minimum required access (least privilege) */
zowe tso issue command   "PERMIT 'PAYMT1P.**' CLASS(DIGTCERT) ID(AUTOBOT) ACCESS(UPDATE)"
zowe tso issue command "SETROPTS RACLIST(DIGTCERT) REFRESH"

/* Step 4: Re-test the failing operation */

/* ServiceNow: Raise a Change Request for the PERMIT command */
/* Automation IDs must never receive access outside of a reviewed CR */
# Attach RLIST output (before) and PERMIT command as evidence

/* SYSTEM PROMPT — P-01 */
You are a z/OS RACF JCL generation assistant operating inside a
policy-gated automation pipeline.

Constraints:
- Output ONLY valid z/OS JCL. No prose, no markdown, no comments outside /* */.
- Permitted verbs: RACDCERT GENCERT, CONNECT, REMOVE, SETROPTS RACLIST(DIGTCERT) REFRESH.
- NOTAFTER must not exceed today + 365 days.
- Ring name must match: {ENV}.{APPID}.{RUNTIME}.{PURPOSE}.
- Always include SETROPTS RACLIST(DIGTCERT) REFRESH as the final SYSTSIN statement.
- Do NOT include DELETE or EXPORT unless the user prompt explicitly says "confirmed delete".

/* USER PROMPT TEMPLATE — P-01 */
Generate a certificate renewal JCL for:
  Owner ID:    {{OWNER_ID}}
  Label:       {{NEW_LABEL}}
  CN:          {{FQDN}}
  OU:          {{OU}}
  O:           {{ORG}}
  C:           {{COUNTRY}}
  Ring:        {{RING_NAME}}
  Valid until: {{NOTAFTER_DATE}}
  Key usage:   {{KEYUSAGE}}
  Replace old label: {{OLD_LABEL}} (connect new as DEFAULT, leave old connected)

/* HOW TO FEED REAL DATA — Zowe CLI */
# Get current cert details to populate the template:
zowe tso issue command "RACDCERT ID(PAYMT1P) LIST(LABEL('P.PAYMT1.TLS.2025'))"   > current_cert.txt
# Pipe to orchestrator which fills the template and calls the LLM API

/* SERVICENOW — where output goes */
# Orchestrator creates Change Request:
#   short_description: "AI Renewal JCL: {{NEW_LABEL}}"
#   u_jcl_content:     <base64 JCL>
#   u_zowe_dataset:    "{{OWNER_ID}}.AI.RENEW.JCL"
#   state:             "pending_approval"

/* SYSTEM PROMPT — P-02 */
You are a structured data extraction assistant for z/OS RACF output.
Extract all certificate records from the RACDCERT LIST spool output provided.

Output a JSON array. Each object must contain exactly these fields
(use null for absent fields — do not omit fields):
  label, owner_id, subject_dn, issuer_dn, not_before, not_after,
  key_type, key_size, serial_number, trust_status, is_self_signed,
  connected_rings (string array)

Output ONLY valid JSON. No prose, no markdown fences, no trailing commas.

/* USER PROMPT TEMPLATE — P-02 */
Extract all certificates from the following RACDCERT LIST output:
<SPOOL_OUTPUT>
{{SPOOL_TEXT}}
</SPOOL_OUTPUT>

/* HOW TO FEED REAL DATA — Zowe CLI */
# Retrieve nightly scan spool:
zowe jobs list jobs --owner CERTSCAN --prefix CERTSCAN | head -1
zowe jobs view spool-file-by-id JOB99001 2 > spool_output.txt

# Call LLM extraction API:
curl -s -X POST $LLM_API_URL   -H "Authorization: Bearer $LLM_API_KEY"   -d "{"system": "<P-02 system prompt>",
       "user": "<P-02 user prompt with spool_output.txt content>"}"   | jq '.result' > extracted_certs.json

/* SERVICENOW — where output goes */
# Orchestrator upserts extracted_certs.json into inventory DB.
# Any cert with not_after < today+60 and no open ticket:
#   → auto-raise Incident via ServiceNow REST API (see Section 11.2)

/* SYSTEM PROMPT — P-03 */
You are a z/OS RACF diagnostic assistant with deep knowledge of
RACF certificate management, AT-TLS, CICS, MQ, and z/OS Connect.

When given an error code and JES/spool output:
1. Identify the root cause from the error code and context.
2. List the exact Zowe CLI commands to diagnose and remediate, in order.
3. State whether a SETROPTS RACLIST(DIGTCERT) REFRESH is required.
4. Draft a ServiceNow work note (2-3 sentences, plain English).
5. Flag if a Security Incident should be raised (e.g., unauthorised change).

Format your response as JSON with keys:
  root_cause, zowe_commands (array), refresh_required (bool),
  snow_work_note, raise_security_incident (bool), confidence (high|medium|low)

/* USER PROMPT TEMPLATE — P-03 */
Error code: {{ERROR_CODE}}
Application: {{APP_ID}}
Ring: {{RING_NAME}}
ServiceNow incident: {{INC_NUMBER}}

JES output:
<JES_OUTPUT>
{{JES_TEXT}}
</JES_OUTPUT>

/* HOW TO FEED REAL DATA — Zowe CLI */
# Get JES output for the failing job:
zowe jobs view spool-file-by-id JOB12345 2 > jes_output.txt

/* SERVICENOW — where output goes */
# Orchestrator posts snow_work_note to the incident:
# curl -X PATCH .../incident/{{INC_NUMBER}} #   -d '{"work_notes": "<snow_work_note from LLM>"}'
# If raise_security_incident == true:
#   Orchestrator creates a linked Security Incident with P1 priority

/* SYSTEM PROMPT — P-04 */
You are a certificate policy validation assistant for a regulated bank.
Validate the submitted certificate request fields against these rules:
  - CN must be a fully qualified domain name ending in .internal.bank.com
  - OU must match an entry in the approved OU list (provided below)
  - O must be exactly "First National Bank"
  - C must be "US"
  - NOTAFTER must not exceed 365 days from today ({{TODAY}})
  - KEYUSAGE must be HANDSHAKE for TLS, DATAENCRYPT for encryption certs
  - Ring name must match {ENV}.{APPID}.{RUNTIME}.{PURPOSE} schema
  - APPID must exist in the CMDB application registry (list provided below)

Approved OUs: {{APPROVED_OU_LIST}}
CMDB app IDs: {{CMDB_APPID_LIST}}

Return JSON: { passed: bool, violations: [{ field, value, rule, message }] }

/* USER PROMPT TEMPLATE — P-04 */
Validate this certificate request:
  Owner ID:  {{OWNER_ID}}
  CN:        {{CN}}
  OU:        {{OU}}
  O:         {{ORG}}
  C:         {{COUNTRY}}
  NOTAFTER:  {{NOTAFTER_DATE}}
  KEYUSAGE:  {{KEYUSAGE}}
  Ring:      {{RING_NAME}}

/* HOW TO FEED REAL DATA — Zowe CLI */
# Fetch live CMDB app IDs for injection into system prompt:
zowe files download data-set "SECURITY.CMDB.APPIDS" > cmdb_appids.txt

/* SERVICENOW — where output goes */
# If passed == false: orchestrator blocks CR creation and returns
#   violations to the engineer in the Virtual Agent chat.
# If passed == true: orchestrator proceeds to P-01 JCL generation
#   and creates the Change Request.

/* ── Define RRSF nodes (run once on each LPAR, by security admin) ── */

/* On LPAR1 (SYSA): define peer node SYSB */
RACDEF RRSF.NODE.SYSB.TCPIP CLASS(FACILITY)
PERMIT RRSF.NODE.SYSB.TCPIP CLASS(FACILITY) ID(RACFUSER) ACCESS(READ)

/* Activate RRSF with AUTODIRECT for certificate operations */
RRSF AUTODIRECT(YES) VERIFYUSER(YES)

/* Verify node status */
RRSF LIST
/* Expected output:
   NODE: SYSB   STATUS: CONNECTED   MODE: AUTODIRECT
   NODE: SYSC   STATUS: CONNECTED   MODE: AUTODIRECT */

/* ── Zowe CLI: check RRSF node status from pipeline ── */
zowe tso issue command "RRSF LIST"
/* Parse output to confirm all nodes are CONNECTED before
   submitting any multi-LPAR certificate change */

/* ── Standard multi-LPAR cert renewal flow with AUTODIRECT ── */

/* Step 1: Issue RACDCERT on primary LPAR (SYSA) — AUTODIRECT replicates */
zowe tso issue command   "RACDCERT ID(PAYMT1P) GENCERT +
   SUBJECTSDN(CN('paymt1.internal.bank.com') O('First National Bank') C('US')) +
   WITHLABEL('P.PAYMT1.TLS.2027') +
   NOTAFTER(DATE(2027/03/05)) +
   KEYUSAGE(HANDSHAKE) +
   SIGNWITH(CERTAUTH LABEL('INTERNAL ROOT CA'))"

/* Step 2: Wait for RRSF propagation (typically 2–10 seconds) */
/* Pipeline: poll with 15-second retry, max 3 attempts */

/* Step 3: Verify cert exists on each LPAR */
zowe tso issue command "RACDCERT ID(PAYMT1P) LIST(LABEL('P.PAYMT1.TLS.2027'))"   --zosmf-host SYSB-ZOSMF.bank.com
zowe tso issue command "RACDCERT ID(PAYMT1P) LIST(LABEL('P.PAYMT1.TLS.2027'))"   --zosmf-host SYSC-ZOSMF.bank.com
/* Each Zowe profile targets a different LPAR's z/OSMF endpoint */

/* Step 4: Run SETROPTS REFRESH on EVERY LPAR */
for LPAR in SYSA SYSB SYSC; do
  zowe tso issue command "SETROPTS RACLIST(DIGTCERT) REFRESH"     --zosmf-host ${LPAR}-ZOSMF.bank.com
done

/* Step 5: Verify ring connection on each LPAR */
for LPAR in SYSA SYSB SYSC; do
  zowe tso issue command     "RACDCERT ID(PAYMT1P) LISTRING(P.PAYMT1.ATTLS.TLS)"     --zosmf-host ${LPAR}-ZOSMF.bank.com
done

# ── Nightly multi-LPAR consistency check ──────────────────────────

for LPAR in SYSA SYSB SYSC; do
  # Retrieve spool from each LPAR's CERTSCAN job
  zowe jobs view spool-file-by-id $(zowe jobs list jobs     --owner CERTSCAN --prefix CERTSCAN     --zosmf-host ${LPAR}-ZOSMF.bank.com     | head -1 | awk '{print $1}') 2     --zosmf-host ${LPAR}-ZOSMF.bank.com     > inventory_${LPAR}.txt

  # Run P-02 LLM extraction per LPAR
  curl -s -X POST $LLM_API_URL     -d "{"prompt": "<P-02 system prompt>", "input": "$(cat inventory_${LPAR}.txt)"}"     | jq '.result' > certs_${LPAR}.json
done

# Diff: find labels on SYSA not present on SYSB or SYSC
python3 - <<'EOF'
import json
with open('certs_SYSA.json') as f: sysa = {c['label'] for c in json.load(f)}
with open('certs_SYSB.json') as f: sysb = {c['label'] for c in json.load(f)}
with open('certs_SYSC.json') as f: sysc = {c['label'] for c in json.load(f)}
gaps = sysa - sysb | sysa - sysc
if gaps:
    print("REPLICATION GAPS:", gaps)  # → trigger ServiceNow incident
EOF

# ServiceNow: raise Incident for each replication gap
# curl -X POST .../incident -d '{
#   "short_description": "RRSF replication gap: <label> missing on <LPAR>",
#   "urgency": "2", "impact": "2",
#   "assignment_group": "RACF Security Engineering"
# }'

# ── HashiCorp Vault: orchestrator secrets retrieval flow ──────────

# 1. Orchestrator authenticates to Vault using AppRole
VAULT_TOKEN=$(curl -s -X POST $VAULT_ADDR/v1/auth/approle/login   -d "{"role_id": "$APPROLE_ROLE_ID",
       "secret_id": "$APPROLE_SECRET_ID"}"   | jq -r '.auth.client_token')

# 2. Retrieve z/OSMF credentials (short-lived, 1-hour TTL)
ZOSMF_CREDS=$(curl -s -H "X-Vault-Token: $VAULT_TOKEN"   $VAULT_ADDR/v1/secret/data/racf-pipeline/zosmf)
ZOSMF_USER=$(echo $ZOSMF_CREDS | jq -r '.data.data.username')
ZOSMF_PASS=$(echo $ZOSMF_CREDS | jq -r '.data.data.password')

# 3. Retrieve ServiceNow OAuth token
SN_TOKEN=$(curl -s -H "X-Vault-Token: $VAULT_TOKEN"   $VAULT_ADDR/v1/secret/data/racf-pipeline/servicenow   | jq -r '.data.data.oauth_token')

# 4. Retrieve LLM API key
LLM_KEY=$(curl -s -H "X-Vault-Token: $VAULT_TOKEN"   $VAULT_ADDR/v1/secret/data/racf-pipeline/llm   | jq -r '.data.data.api_key')

# 5. Configure Zowe profile using retrieved credentials (in-memory only)
zowe config set profiles.racf-prod.properties.user "$ZOSMF_USER" --global-config
zowe config set profiles.racf-prod.properties.password "$ZOSMF_PASS" --global-config

# 6. Run the pipeline (Zowe CLI, ServiceNow REST calls, LLM API)
# ... pipeline steps ...

# 7. Revoke the Vault token after pipeline completes (zero standing access)
curl -s -X POST -H "X-Vault-Token: $VAULT_TOKEN"   $VAULT_ADDR/v1/auth/token/revoke-self

# 8. Clear Zowe in-memory credentials
zowe config delete profiles.racf-prod.properties.password --global-config

# ── CyberArk Conjur: orchestrator secrets retrieval ───────────────

# 1. Authenticate orchestrator host identity to Conjur
CONJUR_TOKEN=$(curl -s -X POST   "$CONJUR_URL/authn/$CONJUR_ACCOUNT/$CONJUR_HOST_ID/authenticate"   --data-binary "$CONJUR_API_KEY"   | base64 | tr -d '
')

# 2. Retrieve z/OSMF password (CPM-managed, auto-rotated)
ZOSMF_PASS=$(curl -s   -H "Authorization: Token token="$CONJUR_TOKEN""   "$CONJUR_URL/secrets/$CONJUR_ACCOUNT/variable/racf-pipeline%2Fzosmf%2Fpassword")

# 3. Retrieve ServiceNow OAuth token
SN_TOKEN=$(curl -s   -H "Authorization: Token token="$CONJUR_TOKEN""   "$CONJUR_URL/secrets/$CONJUR_ACCOUNT/variable/racf-pipeline%2Fsn%2Foauth-token")

# 4. Retrieve LLM API key
LLM_KEY=$(curl -s   -H "Authorization: Token token="$CONJUR_TOKEN""   "$CONJUR_URL/secrets/$CONJUR_ACCOUNT/variable/racf-pipeline%2Fllm%2Fapi-key")

# 5. CyberArk also integrates with ServiceNow via the CyberArk App for ServiceNow:
#    ServiceNow Flow Designer retrieves credentials from CyberArk at runtime
#    using the CyberArk MID Server plugin — no secrets stored in ServiceNow.

# 6. Zowe team profile references CyberArk credential provider:
# zowe.config.json:
# {
#   "profiles": {
#     "racf-prod": {
#       "properties": {
#         "host": "SYSA-ZOSMF.bank.com",
#         "port": 443
#       },
#       "secure": ["user", "password"]  ← resolved from CyberArk at runtime
#     }
#   }
# }

/* ── ServiceNow: vault-backed credential flow ── */

/* 1. ServiceNow Connection Alias (no secrets stored) */
{
  "name":        "RACF_ZOSMF_SYSA",
  "type":        "HTTP",
  "host":        "SYSA-ZOSMF.bank.com",
  "port":        443,
  "credential":  "CyberArk_RACF_ZOSMF"   /* ← references CyberArk, not a password */
}

/* 2. CyberArk App for ServiceNow resolves the credential at runtime:
      ServiceNow Flow Designer → CyberArk MID Server plugin
      → CyberArk Vault → returns current z/OSMF password
      → MID Server uses it for the Zowe CLI call
      → password never written to ServiceNow database */

/* 3. Vault audit log entry (HashiCorp Vault example):
{
  "time":      "2026-03-05T14:32:01Z",
  "type":      "response",
  "auth":      { "display_name": "racf-orchestrator", "policies": ["racf-pipeline"] },
  "request":   { "path": "secret/data/racf-pipeline/zosmf", "operation": "read" },
  "response":  { "data": { "keys": ["username", "password"] } }  /* values redacted */
}
/* Every secret retrieval is logged with: who, what, when, from which IP */