diff --git a/content/code-security/concepts/code-scanning/codeql/codeql-query-packs.md b/content/code-security/concepts/code-scanning/codeql/codeql-query-packs.md index 70f80ffb838e..0ea853f50743 100644 --- a/content/code-security/concepts/code-scanning/codeql/codeql-query-packs.md +++ b/content/code-security/concepts/code-scanning/codeql/codeql-query-packs.md @@ -43,6 +43,8 @@ The standard {% data variables.product.prodname_codeql %} packs for all supporte * `codeql/ruby-queries` * `codeql/swift-queries` +For more information about compatibility between published query packs and different {% data variables.product.prodname_codeql %} releases, see [AUTOTITLE](/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/publishing-and-using-codeql-packs#about-codeql-pack-compatibility). + You can also use the {% data variables.product.prodname_codeql_cli %} to create your own {% data variables.product.prodname_codeql %} packs, add dependencies to packs, and install or update dependencies. You can publish {% data variables.product.prodname_codeql %} packs that you have created, using the {% data variables.product.prodname_codeql_cli %}. For more information on publishing and downloading {% data variables.product.prodname_codeql %} packs, see [AUTOTITLE](/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/publishing-and-using-codeql-packs). diff --git a/content/code-security/concepts/code-scanning/index.md b/content/code-security/concepts/code-scanning/index.md index a1de0ceae8a1..a94806e064d9 100644 --- a/content/code-security/concepts/code-scanning/index.md +++ b/content/code-security/concepts/code-scanning/index.md @@ -17,5 +17,6 @@ children: - /about-code-scanning-alerts - /setup-types - /about-integration-with-code-scanning + - /sarif-files - /codeql --- diff --git a/content/code-security/concepts/code-scanning/sarif-files.md b/content/code-security/concepts/code-scanning/sarif-files.md new file mode 100644 index 000000000000..fca271bbb5ef --- /dev/null +++ b/content/code-security/concepts/code-scanning/sarif-files.md @@ -0,0 +1,31 @@ +--- +title: About SARIF files for code scanning +shortTitle: SARIF files +intro: SARIF files convert third-party analyses into alerts on {% data variables.product.github %}. +topics: + - Code Security + - Code scanning +versions: + fpt: '*' + ghes: '*' + ghec: '*' +contentType: concepts +--- + +>[!NOTE] If you use default setup for {% data variables.product.prodname_code_scanning %}, or an advanced setup that involves using {% data variables.product.prodname_actions %} to run the {% data variables.product.prodname_codeql %} action, then you don't need to interact with SARIF files. Scan results are uploaded and parsed as {% data variables.product.prodname_code_scanning %} alerts automatically. + +SARIF stands for _Static Analysis Results Interchange Format_. This is a JSON-based standard for storing results from static analysis tools. + +If you use a **third-party analysis tool or CI/CD system** to scan code for vulnerabilities, you can generate a SARIF file and upload it to {% data variables.product.github %}. {% data variables.product.github %} will parse the SARIF file and show alerts using the results in your repository as a part of the {% data variables.product.prodname_code_scanning %} experience. + +{% data variables.product.github %} uses properties in the SARIF file to display alerts. For example, the `shortDescription` and `fullDescription` appear at the top of a {% data variables.product.prodname_code_scanning %} alert. The `location` allows {% data variables.product.github %} to show annotations in your code file. + +This article explains how SARIF files are used on {% data variables.product.github %}. If you're new to SARIF and want to learn more, see Microsoft's [`SARIF tutorials`](https://github.com/microsoft/sarif-tutorials) repository. + +## Version requirements + +{% data variables.product.prodname_code_scanning_caps %} supports a subset of the [SARIF 2.1.0](https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html) JSON schema. Ensure that SARIF files from third-party tools use this version. + +## Upload methods + +You can upload a SARIF file using {% data variables.product.prodname_actions %}, the {% data variables.product.prodname_code_scanning %} API, or the {% data variables.product.prodname_codeql_cli %}. The best upload method depends on how you generate the SARIF file. For more information, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/uploading-a-sarif-file-to-github). diff --git a/content/code-security/how-tos/manage-security-alerts/manage-code-scanning-alerts/triaging-code-scanning-alerts-in-pull-requests.md b/content/code-security/how-tos/manage-security-alerts/manage-code-scanning-alerts/triaging-code-scanning-alerts-in-pull-requests.md index 393bfd0b209b..599763816e85 100644 --- a/content/code-security/how-tos/manage-security-alerts/manage-code-scanning-alerts/triaging-code-scanning-alerts-in-pull-requests.md +++ b/content/code-security/how-tos/manage-security-alerts/manage-code-scanning-alerts/triaging-code-scanning-alerts-in-pull-requests.md @@ -35,7 +35,7 @@ If the lines of code changed in the pull request generate {% data variables.prod * The **Files changed** tab of the pull request > [!NOTE] -> {% data variables.product.prodname_code_scanning_caps %} displays alerts in pull requests only when all the lines of code identified by the alert exist in the pull request diff. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#specifying-the-location-for-source-files). +> {% data variables.product.prodname_code_scanning_caps %} displays alerts in pull requests only when all the lines of code identified by the alert exist in the pull request diff. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#source-file-locations). {% ifversion code-scanning-autofix %} diff --git a/content/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/uploading-a-sarif-file-to-github.md b/content/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/uploading-a-sarif-file-to-github.md index 9ea7803d9507..ab8730af625b 100644 --- a/content/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/uploading-a-sarif-file-to-github.md +++ b/content/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/uploading-a-sarif-file-to-github.md @@ -28,18 +28,16 @@ contentType: how-tos {% data reusables.code-scanning.enterprise-enable-code-scanning %} -## About SARIF file uploads for {% data variables.product.prodname_code_scanning %} +If you use a third-party analysis tool or CI/CD system to scan code for vulnerabilities, you can generate SARIF file and upload it to {% data variables.product.github %}. The best upload method depends on how you generate the SARIF file. -{% data variables.product.prodname_dotcom %} creates {% data variables.product.prodname_code_scanning %} alerts in a repository using information from Static Analysis Results Interchange Format (SARIF) files. SARIF files can be uploaded to a repository using the API or {% data variables.product.prodname_actions %}. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/using-code-scanning-with-your-existing-ci-system). - -You can generate SARIF files using many static analysis security testing tools, including {% data variables.product.prodname_codeql %}. The results must use SARIF version 2.1.0. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning). - -You can upload the results using {% data variables.product.prodname_actions %}, the {% data variables.product.prodname_code_scanning %} API, or the {% data variables.product.prodname_codeql_cli %}. The best upload method will depend on how you generate the SARIF file, for example, if you use: +For example, if you use: * {% data variables.product.prodname_actions %} to run the {% data variables.product.prodname_codeql %} action, there is no further action required. The {% data variables.product.prodname_codeql %} action uploads the SARIF file automatically when it completes analysis. -* {% data variables.product.prodname_actions %} to run a SARIF-compatible analysis tool, you could update the workflow to include a final step that uploads the results (see below). -* The {% data variables.product.prodname_codeql_cli %} to run {% data variables.product.prodname_code_scanning %} in your CI system, you can use the CLI to upload results to {% data variables.product.prodname_dotcom %} (for more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/using-code-scanning-with-your-existing-ci-system)). -* A tool that generates results as an artifact outside of your repository, you can use the {% data variables.product.prodname_code_scanning %} API to upload the file (for more information, see [AUTOTITLE](/rest/code-scanning/code-scanning#upload-an-analysis-as-sarif-data)). +* {% data variables.product.prodname_actions %} to run a SARIF-compatible analysis tool, you could update the workflow to include a final step that uploads the results. See [Uploading a {% data variables.product.prodname_code_scanning %} analysis with {% data variables.product.prodname_actions %}](#uploading-a-code-scanning-analysis-with-github-actions). +* The {% data variables.product.prodname_codeql_cli %} to run {% data variables.product.prodname_code_scanning %} in your CI system, you can use the CLI to upload results to {% data variables.product.prodname_dotcom %}. See [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/using-code-scanning-with-your-existing-ci-system). +* A tool that generates results as an artifact outside of your repository, you can use the {% data variables.product.prodname_code_scanning %} API to upload the file. See [AUTOTITLE](/rest/code-scanning/code-scanning#upload-an-analysis-as-sarif-data). + +By default, {% data variables.product.prodname_code_scanning %} expects one SARIF results file per analysis for a repository. If you want to upload more than one set of results for a commit in a repository, you must identify each set of results as a unique set. {% ifversion fpt or ghec %} @@ -61,7 +59,7 @@ For more information, see the [`upload-sarif` action](https://github.com/github/ The `upload-sarif` action can be configured to run when the `push` and `scheduled` event occur. For more information about {% data variables.product.prodname_actions %} events, see [AUTOTITLE](/actions/using-workflows/events-that-trigger-workflows). -If your SARIF file doesn't include `partialFingerprints`, the `upload-sarif` action will calculate the `partialFingerprints` field for you and attempt to prevent duplicate alerts. {% data variables.product.prodname_dotcom %} can only create `partialFingerprints` when the repository contains both the SARIF file and the source code used in the static analysis. For more information about preventing duplicate alerts, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#providing-data-to-track-code-scanning-alerts-across-runs). +If your SARIF file doesn't include `partialFingerprints`, the `upload-sarif` action will calculate the `partialFingerprints` field for you and attempt to prevent duplicate alerts. {% data variables.product.prodname_dotcom %} can only create `partialFingerprints` when the repository contains both the SARIF file and the source code used in the static analysis. For more information about preventing duplicate alerts, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#data-for-preventing-duplicated-alerts). {% data reusables.code-scanning.upload-sarif-alert-limit %} @@ -150,9 +148,21 @@ jobs: sarif_file: results.sarif ``` +## Uploading more than one SARIF file for a commit + +By default, {% data variables.product.prodname_code_scanning %} expects one SARIF results file per analysis for a repository. Consequently, when you upload a second SARIF results file for a commit, it is treated as a replacement for the original set of data. You may want to upload two different SARIF files for one analysis if, for example, your analysis tool generates a different SARIF file for each language it analyzes or each set of rules it uses. If you want to upload more than one set of results for a commit in a repository, you must identify each set of results as a unique set. + +When you upload multiple SARIF files for a commit, you must indicate a "category" for each analysis. The way to specify a category varies according to the analysis method: +* Using the {% data variables.product.prodname_codeql_cli %} directly, pass the `--sarif-category` argument to the `codeql database analyze` command when you generate SARIF files. For more information, see [AUTOTITLE](/code-security/codeql-cli/getting-started-with-the-codeql-cli/about-the-codeql-cli#about-generating-code-scanning-results-with-the-codeql-cli). +* Using {% data variables.product.prodname_actions %} with `codeql-action/analyze`, the category is set automatically from the workflow name and any matrix variables (typically, `language`). You can override this by specifying a `category` input for the action, which is useful when you analyze different sections of a monorepo in a single workflow. +* Using {% data variables.product.prodname_actions %} to upload results from other static analysis tools, then you must specify a `category` input if you upload more than one file of results for the same tool in one workflow. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/uploading-a-sarif-file-to-github#uploading-a-code-scanning-analysis-with-github-actions). +* If you are not using either of these approaches, you must specify a unique `runAutomationDetails.id` in each SARIF file to upload. For more information about this property, see [`runAutomationDetails` object](#runautomationdetails-object). + +If you upload a second SARIF file for a commit with the same category and from the same tool, the earlier results are overwritten. However, if you try to upload multiple SARIF files for the same tool and category in a single {% data variables.product.prodname_actions %} workflow run, the misconfiguration is detected and the run will fail. + ## Further reading -* [AUTOTITLE](/code-security/code-scanning/troubleshooting-sarif-uploads) +* [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads) * [AUTOTITLE](/actions/using-workflows/workflow-syntax-for-github-actions) * [AUTOTITLE](/actions/monitoring-and-troubleshooting-workflows/viewing-workflow-run-history) * [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/using-code-scanning-with-your-existing-ci-system) diff --git a/content/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/using-code-scanning-with-your-existing-ci-system.md b/content/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/using-code-scanning-with-your-existing-ci-system.md index 55cfbf228436..0bf159dc15e3 100644 --- a/content/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/using-code-scanning-with-your-existing-ci-system.md +++ b/content/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/using-code-scanning-with-your-existing-ci-system.md @@ -59,4 +59,4 @@ Once you have analyzed your code, produced SARIF results, and ensured you can au For specific details on uploading your results to {% data variables.product.github %} using the {% data variables.product.prodname_codeql_cli %}, see [AUTOTITLE](/code-security/codeql-cli/getting-started-with-the-codeql-cli/uploading-codeql-analysis-results-to-github). -By default, {% data variables.product.prodname_code_scanning %} expects one SARIF results file per analysis for a repository. Consequently, when you upload a second SARIF results file for a commit, it is treated as a replacement for the original set of data. You may want to upload two different SARIF files for one analysis if, for example, your analysis tool generates a different SARIF file for each language it analyzes or each set of rules it uses. If you want to upload more than one set of results for a commit in a repository, you must identify each set of results as a unique set. The way to specify a category for a SARIF upload varies according to the analysis method. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#uploading-more-than-one-sarif-file-for-a-commit). +By default, {% data variables.product.prodname_code_scanning %} expects one SARIF results file per analysis for a repository. Consequently, when you upload a second SARIF results file for a commit, it is treated as a replacement for the original set of data. You may want to upload two different SARIF files for one analysis if, for example, your analysis tool generates a different SARIF file for each language it analyzes or each set of rules it uses. If you want to upload more than one set of results for a commit in a repository, you must identify each set of results as a unique set. The way to specify a category for a SARIF upload varies according to the analysis method. diff --git a/content/code-security/how-tos/scan-code-for-vulnerabilities/manage-your-configuration/about-the-tool-status-page.md b/content/code-security/how-tos/scan-code-for-vulnerabilities/manage-your-configuration/about-the-tool-status-page.md index 3579b562a0f5..81afbbaecf6b 100644 --- a/content/code-security/how-tos/scan-code-for-vulnerabilities/manage-your-configuration/about-the-tool-status-page.md +++ b/content/code-security/how-tos/scan-code-for-vulnerabilities/manage-your-configuration/about-the-tool-status-page.md @@ -114,4 +114,4 @@ For integrated tools such as {% data variables.product.prodname_codeql %}, you c > [!NOTE] > If you have set up {% data variables.product.prodname_codeql %} using advanced setup and then set up default setup on the same repository, the {% data variables.code-scanning.tool_status_page %} will only show default setup. -For more information, see [AUTOTITLE](/code-security/code-scanning/troubleshooting-code-scanning) and [AUTOTITLE](/code-security/code-scanning/troubleshooting-sarif-uploads). +For more information, see [AUTOTITLE](/code-security/code-scanning/troubleshooting-code-scanning) and [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads). diff --git a/content/code-security/how-tos/scan-code-for-vulnerabilities/manage-your-configuration/set-code-scanning-merge-protection.md b/content/code-security/how-tos/scan-code-for-vulnerabilities/manage-your-configuration/set-code-scanning-merge-protection.md index c938f77c5ec8..9c38db1b9b6a 100644 --- a/content/code-security/how-tos/scan-code-for-vulnerabilities/manage-your-configuration/set-code-scanning-merge-protection.md +++ b/content/code-security/how-tos/scan-code-for-vulnerabilities/manage-your-configuration/set-code-scanning-merge-protection.md @@ -21,7 +21,7 @@ contentType: how-tos > [!NOTE] > * Merge protection with rulesets is not related to status checks. For more information about status checks, see [AUTOTITLE](/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks). > * Merge protection with rulesets will not apply to merge queue groups or {% data variables.product.prodname_dependabot %} pull requests analyzed by default setup. -> * All the lines of code identified by an alert must exist in the pull request diff. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#specifying-the-location-for-source-files). +> * All the lines of code identified by an alert must exist in the pull request diff. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#source-file-locations). You can use rulesets to prevent pull requests from being merged when one of the following conditions is met: diff --git a/content/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/file-too-large.md b/content/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/file-too-large.md index dc586bc5fdc2..1a494babdf67 100644 --- a/content/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/file-too-large.md +++ b/content/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/file-too-large.md @@ -31,7 +31,7 @@ A fatal error occurred: SARIF file is too large One of these errors is reported if a process attempts to upload a SARIF file that is larger than the maximum size of 10 MB. {% data variables.product.prodname_code_scanning_caps %} does not accept files above this size. There are several different ways to reduce the number of results generated for upload to {% data variables.product.prodname_code_scanning %}. -You could see this error for SARIF files generated by {% data variables.product.prodname_codeql %} or by third-party analysis tools. For information about the limits on uploads, see {% data variables.product.prodname_code_scanning %}, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#validating-your-sarif-file). +You could see this error for SARIF files generated by {% data variables.product.prodname_codeql %} or by third-party analysis tools. For information about the limits on uploads, see {% data variables.product.prodname_code_scanning %}, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#file-compatibility). ## Confirming the cause of the error diff --git a/content/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/results-exceed-limit.md b/content/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/results-exceed-limit.md index bea39feb7005..75ce98347f08 100644 --- a/content/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/results-exceed-limit.md +++ b/content/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/results-exceed-limit.md @@ -48,7 +48,7 @@ You could see these errors for SARIF files generated by {% data variables.produc {% data reusables.code-scanning.sarif-limits %} -For information about validating your SARIF file, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#validating-your-sarif-file). +For information about validating your SARIF file, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#file-compatibility). ## Fixing soft limit errors @@ -56,7 +56,7 @@ When soft limits are exceeded, {% data variables.product.prodname_code_scanning ## Fixing "Analysis SARIF file rejected due to results limits" -There are many considerations and potential solutions for reducing the number of results included in a SARIF results file. For guidance, see [AUTOTITLE](/code-security/code-scanning/troubleshooting-sarif-uploads/file-too-large). +There are many considerations and potential solutions for reducing the number of results included in a SARIF results file. For guidance, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/file-too-large). ## Fixing "Alert(s) in SARIF file exceeded thread flow location limits" @@ -66,24 +66,24 @@ You can configure the analysis to limit the number of dataflow paths included in ## Fixing "Analysis SARIF file rejected due to run limits" -The simplest approach is generate a new SARIF file for each run and upload each file separately. You add a "category" to each results and this enables {% data variables.product.prodname_code_scanning %} to store and display the results appropriately. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#uploading-more-than-one-sarif-file-for-a-commit). +The simplest approach is generate a new SARIF file for each run and upload each file separately. You add a "category" to each results and this enables {% data variables.product.prodname_code_scanning %} to store and display the results appropriately. For more information, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/uploading-a-sarif-file-to-github#uploading-more-than-one-sarif-file-for-a-commit). ## Fixing "Analysis SARIF file rejected due to rule limits" There are two possible approaches here. -1. Reduce the number of rules you use to analyze the code. For more information, see [Defining the query suite to run](/code-security/code-scanning/troubleshooting-sarif-uploads/file-too-large#defining-the-query-suite-to-run) and [Excluding a query from analysis](/code-security/code-scanning/troubleshooting-sarif-uploads/file-too-large#excluding-a-query-from-analysis) in "SARIF results file too large." -1. Run the analysis twice, each time with a different set of rules, and upload both results files to {% data variables.product.prodname_code_scanning %}. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#uploading-more-than-one-sarif-file-for-a-commit). +1. Reduce the number of rules you use to analyze the code. For more information, see [Defining the query suite to run](/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/file-too-large#defining-the-query-suite-to-run) and [Excluding a query from analysis](/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/file-too-large#excluding-a-query-from-analysis) in "SARIF results file too large." +1. Run the analysis twice, each time with a different set of rules, and upload both results files to {% data variables.product.prodname_code_scanning %}. For more information, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/uploading-a-sarif-file-to-github#uploading-more-than-one-sarif-file-for-a-commit). ## Fixing "Analysis SARIF file rejected due to extension limits" -The simplest approach is to create a separate SARIF file each time you run the tool and upload each file separately. You may also need to contact the maintainer of the tool. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#uploading-more-than-one-sarif-file-for-a-commit). +The simplest approach is to create a separate SARIF file each time you run the tool and upload each file separately. You may also need to contact the maintainer of the tool. For more information, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/integrate-with-existing-tools/uploading-a-sarif-file-to-github#uploading-more-than-one-sarif-file-for-a-commit). {% data variables.product.prodname_codeql %} analysis should not generate this error. If you see this error while using the {% data variables.product.prodname_codeql %} action or {% data variables.product.prodname_codeql_cli %}, you should contact {% data variables.contact.github_support %} to let us know. For more information, see [AUTOTITLE](/support/contacting-github-support). ## Fixing "Analysis SARIF file rejected due to location limit" -The best way to resolve this problem is usually to identify the query that reports too many locations and exclude it from analysis. For information on how to do this, see [AUTOTITLE](/code-security/code-scanning/troubleshooting-sarif-uploads/file-too-large#excluding-a-query-from-analysis). +The best way to resolve this problem is usually to identify the query that reports too many locations and exclude it from analysis. For information on how to do this, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/file-too-large#excluding-a-query-from-analysis). ## Fixing "Analysis SARIF file rejected due to rule tag limits" diff --git a/content/code-security/how-tos/secure-at-scale/configure-enterprise-security/establish-complete-coverage/configuring-additional-secret-scanning-settings-for-your-enterprise.md b/content/code-security/how-tos/secure-at-scale/configure-enterprise-security/establish-complete-coverage/configuring-additional-secret-scanning-settings-for-your-enterprise.md index f60611301231..632b0c16941a 100644 --- a/content/code-security/how-tos/secure-at-scale/configure-enterprise-security/establish-complete-coverage/configuring-additional-secret-scanning-settings-for-your-enterprise.md +++ b/content/code-security/how-tos/secure-at-scale/configure-enterprise-security/establish-complete-coverage/configuring-additional-secret-scanning-settings-for-your-enterprise.md @@ -54,6 +54,9 @@ You can customize which secret patterns are included in push protection, giving 1. Under "Additional settings", in the "{% data variables.product.UI_secret_protection_scanning %}" section, click anywhere inside the "Pattern configurations for push protection" row. 1. In the page that gets displayed, make the desired changes in the "Enterprise setting" column. + {% data reusables.secret-scanning.pattern-enablement-org-enterprise %} +For more information on how to read data on the {% data variables.product.prodname_secret_scanning %} pattern configuration page, see [AUTOTITLE](/code-security/reference/secret-security/secret-scanning-pattern-configuration-data). + {% endif %} diff --git a/content/code-security/how-tos/secure-at-scale/configure-organization-security/configure-specific-tools/configuring-advanced-setup-for-code-scanning-with-codeql-at-scale.md b/content/code-security/how-tos/secure-at-scale/configure-organization-security/configure-specific-tools/configuring-advanced-setup-for-code-scanning-with-codeql-at-scale.md index b60b8af759d7..7f5b9adfa693 100644 --- a/content/code-security/how-tos/secure-at-scale/configure-organization-security/configure-specific-tools/configuring-advanced-setup-for-code-scanning-with-codeql-at-scale.md +++ b/content/code-security/how-tos/secure-at-scale/configure-organization-security/configure-specific-tools/configuring-advanced-setup-for-code-scanning-with-codeql-at-scale.md @@ -1,7 +1,7 @@ --- title: Configuring advanced setup for code scanning with CodeQL at scale shortTitle: CodeQL advanced setup at scale -intro: You can use a script to configure advanced setup for {% data variables.product.prodname_code_scanning %} for a specific group of repositories in your organization. +intro: Establish a highly customizable {% data variables.product.prodname_code_scanning %} setup at scale with a script. permissions: '{% data reusables.permissions.security-org-enable %}' product: '{% data reusables.gated-features.code-scanning %}' redirect_from: @@ -18,18 +18,12 @@ allowTitleToDifferFromFilename: true contentType: how-tos --- -## About enabling advanced setup for {% data variables.product.prodname_code_scanning %} with {% data variables.product.prodname_codeql %} at scale - -If you need to configure a highly customizable {% data variables.product.prodname_code_scanning %} setup for many repositories in your organization, or if repositories in your organization are ineligible for default setup, you can enable {% data variables.product.prodname_code_scanning %} at scale with advanced setup. - -To enable advanced setup across multiple repositories, you can write a bulk configuration script. To successfully execute the script, {% data variables.product.prodname_actions %} must be enabled for the {% ifversion fpt %}organization{% elsif ghec %}organization or enterprise{% elsif ghes %}site{% endif %}. - -Alternatively, if you do not need granular control over the {% data variables.product.prodname_code_scanning %} configuration for many repositories in your organization, you can quickly and easily configure {% data variables.product.prodname_code_scanning %} at scale with default setup. For more information, see [AUTOTITLE](/code-security/code-scanning/enabling-code-scanning/configuring-default-setup-for-code-scanning-at-scale). - ## Using a script to enable advanced setup For repositories that are not eligible for default setup, you can use a bulk configuration script to enable advanced setup across multiple repositories. +> [!NOTE] To successfully execute the script, {% data variables.product.prodname_actions %} must be enabled for the {% ifversion fpt %}organization{% elsif ghec %}organization or enterprise{% elsif ghes %}instance{% endif %}. + 1. Identify a group of repositories that can be analyzed using the same {% data variables.product.prodname_code_scanning %} configuration. For example, all repositories that build Java artifacts using the production environment. 1. Create and test a {% data variables.product.prodname_actions %} workflow to call the {% data variables.product.prodname_codeql %} action with the appropriate configuration. For more information, see [AUTOTITLE](/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-advanced-setup-for-code-scanning#configuring-advanced-setup-for-code-scanning-with-codeql). 1. Use one of the example scripts or create a custom script to add the workflow to each repository in the group. @@ -38,10 +32,8 @@ For repositories that are not eligible for default setup, you can use a bulk con * NodeJS example: [`nickliffen/ghas-enablement`](https://github.com/NickLiffen/ghas-enablement) repository * PowerShell example: [`jhutchings1/Create-ActionsPRs`](https://github.com/jhutchings1/Create-ActionsPRs) repository -### Extending {% data variables.product.prodname_codeql %} coverage with model packs +## Next steps {% data reusables.code-scanning.beta-model-packs %} If your codebase depends on a library or framework that is not recognized by the standard queries in {% data variables.product.prodname_codeql %}, you can extend the {% data variables.product.prodname_codeql %} coverage in your bulk configuration script by specifying published {% data variables.product.prodname_codeql %} model packs. For more information, see [AUTOTITLE](/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning#extending-codeql-coverage-with-codeql-model-packs). - -Alternatively, if you do not need granular control over the {% data variables.product.prodname_code_scanning %} configuration for many repositories in your organization, you can quickly and easily configure model packs with {% data variables.product.prodname_code_scanning %} at scale with default setup. For more information, see [AUTOTITLE](/code-security/code-scanning/managing-your-code-scanning-configuration/editing-your-configuration-of-default-setup#extending-codeql-coverage-with-codeql-model-packs-in-default-setup). diff --git a/content/code-security/how-tos/secure-at-scale/configure-organization-security/establish-complete-coverage/configuring-global-security-settings-for-your-organization.md b/content/code-security/how-tos/secure-at-scale/configure-organization-security/establish-complete-coverage/configuring-global-security-settings-for-your-organization.md index a3ef7220b029..f004b445b5bf 100644 --- a/content/code-security/how-tos/secure-at-scale/configure-organization-security/establish-complete-coverage/configuring-global-security-settings-for-your-organization.md +++ b/content/code-security/how-tos/secure-at-scale/configure-organization-security/establish-complete-coverage/configuring-global-security-settings-for-your-organization.md @@ -1,7 +1,7 @@ --- title: Configuring global security settings for your organization shortTitle: Configure global settings -intro: Customize {% data variables.product.prodname_AS %} features to strengthen the security of your organization. +intro: Customize {% data variables.product.prodname_AS %} features for your organization by defining global settings that ensure consistent security standards and safeguard all your repositories. permissions: '{% data reusables.permissions.security-org-enable %}' versions: feature: security-configurations @@ -15,10 +15,6 @@ redirect_from: contentType: how-tos --- -## About {% data variables.product.prodname_global_settings %} - -Alongside {% data variables.product.prodname_security_configurations %}, which determine repository-level security settings, you should also configure {% data variables.product.prodname_global_settings %} for your organization. {% data variables.product.prodname_global_settings_caps %} apply to your entire organization, and can customize {% data variables.product.prodname_AS %} features based on your needs. {% ifversion ghes < 3.16 %}You can also create a team of security managers to monitor and maintain your organization's security.{% endif %} - ## Accessing the {% data variables.product.prodname_global_settings %} page for your organization {% data reusables.profile.access_org %} @@ -27,8 +23,6 @@ Alongside {% data variables.product.prodname_security_configurations %}, which d ## Configuring global {% data variables.product.prodname_dependabot %} settings -{% data reusables.dependabot.dependabot-overview %} - You can customize several {% data variables.product.prodname_global_settings %} for {% data variables.product.prodname_dependabot %}: * [Creating and managing {% data variables.dependabot.auto_triage_rules %}](#creating-and-managing-dependabot-auto-triage-rules) @@ -44,11 +38,11 @@ You can create and manage {% data variables.dependabot.auto_triage_rules %} to i * You can create a new rule by clicking **New rule**, then entering the details for your rule and clicking **Create rule**. * You can edit an existing rule by clicking {% octicon "pencil" aria-label="Edit CURATED-OR-CUSTOM rule" %}, then making the desired changes and clicking **Save rule**. -For more information on {% data variables.dependabot.auto_triage_rules %}, see [AUTOTITLE](/code-security/dependabot/dependabot-auto-triage-rules/about-dependabot-auto-triage-rules) and [AUTOTITLE](/code-security/dependabot/dependabot-auto-triage-rules/customizing-auto-triage-rules-to-prioritize-dependabot-alerts#adding-custom-auto-triage-rules-to-your-organization). +For more information on {% data variables.dependabot.auto_triage_rules %}, see [AUTOTITLE](/code-security/concepts/supply-chain-security/about-dependabot-auto-triage-rules) and [AUTOTITLE](/code-security/how-tos/secure-your-supply-chain/manage-your-dependency-security/customizing-auto-triage-rules-to-prioritize-dependabot-alerts#adding-custom-auto-triage-rules-to-your-organization). ### Grouping {% data variables.product.prodname_dependabot_security_updates %} -{% data variables.product.prodname_dependabot %} can group all automatically suggested security updates into a single pull request. To enable grouped security updates, select **Grouped security updates**. For more information about grouped updates and customization options, see [AUTOTITLE](/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates#grouping-dependabot-security-updates-into-a-single-pull-request). +{% data variables.product.prodname_dependabot %} can group all automatically suggested security updates into a single pull request. To enable grouped security updates, select **Grouped security updates**. For more information about grouped updates and customization options, see [AUTOTITLE](/code-security/how-tos/secure-your-supply-chain/secure-your-dependencies/configuring-dependabot-security-updates#grouping-dependabot-security-updates-into-a-single-pull-request). {% ifversion dependabot-on-actions-opt-in %} @@ -93,7 +87,7 @@ For more information about configuring self-hosted runners for {% data variables ### Granting {% data variables.product.prodname_dependabot %} access to private {% ifversion ghec or ghes %}and internal {% endif %}repositories -To update private dependencies of repositories in your organization, {% data variables.product.prodname_dependabot %} needs access to those repositories. To grant {% data variables.product.prodname_dependabot %} access to the desired private {% ifversion ghec or ghes %}or internal {% endif %}repository, scroll down to the "Grant {% data variables.product.prodname_dependabot %} access to private repositories" section, then use the search bar to find and select the desired repository. Be aware that granting {% data variables.product.prodname_dependabot %} access to a repository means all users in your organization will have access to the contents of that repository through {% data variables.product.prodname_dependabot_updates %}. For more information about the supported ecosystems for private repositories, see [AUTOTITLE](/code-security/dependabot/ecosystems-supported-by-dependabot/supported-ecosystems-and-repositories). +To update private dependencies of repositories in your organization, {% data variables.product.prodname_dependabot %} needs access to those repositories. To grant {% data variables.product.prodname_dependabot %} access to the desired private {% ifversion ghec or ghes %}or internal {% endif %}repository, scroll down to the "Grant {% data variables.product.prodname_dependabot %} access to private repositories" section, then use the search bar to find and select the desired repository. Be aware that granting {% data variables.product.prodname_dependabot %} access to a repository means all users in your organization will have access to the contents of that repository through {% data variables.product.prodname_dependabot_updates %}. For more information about the supported ecosystems for private repositories, see [AUTOTITLE](/code-security/reference/supply-chain-security/supported-ecosystems-and-repositories). ## Configuring global {% data variables.product.prodname_code_scanning %} settings @@ -117,7 +111,7 @@ You can customize several {% data variables.product.prodname_global_settings %} ### Recommending the extended query suite for default setup -{% data variables.product.prodname_code_scanning_caps %} offers specific groups of {% data variables.product.prodname_codeql %} queries, called {% data variables.product.prodname_codeql %} query suites, to run against your code. By default, the "Default" query suite is run. {% data variables.product.company_short %} also offers the "Extended" query suite, which contains all the queries in the "Default" query suite, plus additional queries with lower precision and severity. To suggest the "Extended" query suite across your organization, select **Recommend the extended query suite for repositories enabling default setup**. For more information on built-in query suites for {% data variables.product.prodname_codeql %} default setup, see [AUTOTITLE](/code-security/code-scanning/managing-your-code-scanning-configuration/codeql-query-suites). +{% data variables.product.prodname_code_scanning_caps %} offers specific groups of {% data variables.product.prodname_codeql %} queries, called {% data variables.product.prodname_codeql %} query suites, to run against your code. By default, the "Default" query suite is run. {% data variables.product.company_short %} also offers the "Extended" query suite, which contains all the queries in the "Default" query suite, plus additional queries with lower precision and severity. To suggest the "Extended" query suite across your organization, select **Recommend the extended query suite for repositories enabling default setup**. For more information on built-in query suites for {% data variables.product.prodname_codeql %} default setup, see [AUTOTITLE](/code-security/concepts/code-scanning/codeql/codeql-query-suites). {% ifversion code-scanning-autofix %} @@ -129,7 +123,7 @@ You can select **{% data variables.copilot.copilot_autofix_short %}** to enable ### Expanding {% data variables.product.prodname_codeql %} analysis -You can expand {% data variables.product.prodname_codeql %} analysis coverage for all repositories in your organization that use default setup by configuring {% data variables.product.prodname_codeql %} model packs. Model packs extend the {% data variables.product.prodname_codeql %} analysis to recognize additional frameworks and libraries that are not included in the standard {% data variables.product.prodname_codeql %} libraries. This global configuration applies to repositories using default setup and allows you to specify model packs published via the container registry. For more information, see [AUTOTITLE](/code-security/code-scanning/managing-your-code-scanning-configuration/editing-your-configuration-of-default-setup#extending-coverage-for-all-repositories-in-an-organization). +You can expand {% data variables.product.prodname_codeql %} analysis coverage for all repositories in your organization that use default setup by configuring {% data variables.product.prodname_codeql %} model packs. Model packs extend the {% data variables.product.prodname_codeql %} analysis to recognize additional frameworks and libraries that are not included in the standard {% data variables.product.prodname_codeql %} libraries. This global configuration applies to repositories using default setup and allows you to specify model packs published via the container registry. For more information, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/manage-your-configuration/editing-your-configuration-of-default-setup#extending-coverage-for-all-repositories-in-an-organization). {% ifversion ghes < 3.17 %} @@ -155,7 +149,7 @@ To provide context for developers when {% data variables.product.prodname_secret ### Defining custom patterns -You can define custom patterns for {% data variables.product.prodname_secret_scanning %} with regular expressions. Custom patterns can identify secrets that are not detected by the default patterns supported by {% data variables.product.prodname_secret_scanning %}. To create a custom pattern, click **New pattern**, then enter the details for your pattern and click **Save and dry run**. For more information on custom patterns, see [AUTOTITLE](/code-security/secret-scanning/using-advanced-secret-scanning-and-push-protection-features/custom-patterns/defining-custom-patterns-for-secret-scanning). +You can define custom patterns for {% data variables.product.prodname_secret_scanning %} with regular expressions. Custom patterns can identify secrets that are not detected by the default patterns supported by {% data variables.product.prodname_secret_scanning %}. To create a custom pattern, click **New pattern**, then enter the details for your pattern and click **Save and dry run**. For more information on custom patterns, see [AUTOTITLE](/code-security/how-tos/secure-your-secrets/customize-leak-detection/defining-custom-patterns-for-secret-scanning). {% ifversion push-protected-pattern-configuration %} @@ -167,8 +161,11 @@ You can customize which secret patterns are included in push protection, giving 1. Under "Additional settings", in the "{% data variables.product.prodname_secret_scanning_caps %}" section and to the right of "Pattern configurations", click **{% octicon "gear" aria-label="The Gear icon" %}**. 1. In the page that gets displayed, make the desired changes in the "Organization setting" column. + {% data reusables.secret-scanning.pattern-enablement-org-enterprise %} +For more information on how to read data on the {% data variables.product.prodname_secret_scanning %} pattern configuration page, see [AUTOTITLE](/code-security/reference/secret-security/secret-scanning-pattern-configuration-data). + {% endif %} ## Creating security managers for your organization diff --git a/content/code-security/reference/code-scanning/sarif-support-for-code-scanning.md b/content/code-security/reference/code-scanning/sarif-support-for-code-scanning.md index 747578b4ad4b..322550b6ed43 100644 --- a/content/code-security/reference/code-scanning/sarif-support-for-code-scanning.md +++ b/content/code-security/reference/code-scanning/sarif-support-for-code-scanning.md @@ -1,7 +1,7 @@ --- title: SARIF support for code scanning shortTitle: SARIF support -intro: To display results from a third-party static analysis tool in your repository on {% data variables.product.prodname_dotcom %}, you'll need your results stored in a SARIF file that supports a specific subset of the SARIF 2.1.0 JSON schema for {% data variables.product.prodname_code_scanning %}. If you use the default {% data variables.product.prodname_codeql %} static analysis engine, then your results will display in your repository on {% data variables.product.prodname_dotcom %} automatically. +intro: Ensure your SARIF files from third-party tools comply with {% data variables.product.github %}'s standards. product: '{% data reusables.gated-features.code-scanning %}' redirect_from: - /github/finding-security-vulnerabilities-and-errors-in-your-code/about-sarif-support-for-code-scanning @@ -22,27 +22,21 @@ topics: contentType: reference --- -## About SARIF support +{% data variables.product.github %} can parse SARIF files produced by third-party tools to display {% data variables.product.prodname_code_scanning %} alerts in repositories. For more information, see [AUTOTITLE](/code-security/concepts/code-scanning/sarif-files). -SARIF (Static Analysis Results Interchange Format) is an [OASIS Standard](https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html) that defines an output file format. The SARIF standard is used to streamline how static analysis tools share their results. {% data variables.product.prodname_code_scanning_caps %} supports a subset of the SARIF 2.1.0 JSON schema. +This article highlights key properties of SARIF files to help you upload SARIF files that meet {% data variables.product.github %}'s requirements and can be converted into useful alerts. -To upload a SARIF file from a third-party static code analysis engine, you'll need to ensure that uploaded files use the SARIF 2.1.0 version. {% data variables.product.prodname_dotcom %} will parse the SARIF file and show alerts using the results in your repository as a part of the {% data variables.product.prodname_code_scanning %} experience. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/uploading-a-sarif-file-to-github). For more information about the SARIF 2.1.0 JSON schema, see [`sarif-schema-2.1.0.json`](https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json). +If you're using {% data variables.product.prodname_actions %} with the {% data variables.code-scanning.codeql_workflow %} or using the {% data variables.product.prodname_codeql_cli %}, then the {% data variables.product.prodname_code_scanning %} results will automatically use the supported subset of SARIF 2.1.0. -If you're using {% data variables.product.prodname_actions %} with the {% data variables.code-scanning.codeql_workflow %} or using the {% data variables.product.prodname_codeql_cli %}, then the {% data variables.product.prodname_code_scanning %} results will automatically use the supported subset of SARIF 2.1.0. For more information, see [AUTOTITLE](/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/configuring-advanced-setup-for-code-scanning) or [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/using-code-scanning-with-your-existing-ci-system). - -{% data variables.product.prodname_dotcom %} uses properties in the SARIF file to display alerts. For example, the `shortDescription` and `fullDescription` appear at the top of a {% data variables.product.prodname_code_scanning %} alert. The `location` allows {% data variables.product.prodname_dotcom %} to show annotations in your code file. For more information, see [AUTOTITLE](/code-security/code-scanning/managing-code-scanning-alerts/about-code-scanning-alerts). - -If you're new to SARIF and want to learn more, see Microsoft's [`SARIF tutorials`](https://github.com/microsoft/sarif-tutorials) repository. - -## Providing data to track {% data variables.product.prodname_code_scanning %} alerts across runs +## Data for preventing duplicated alerts Each time the results of a new code scan are uploaded, the results are processed and alerts are added to the repository. To prevent duplicate alerts for the same problem, {% data variables.product.prodname_code_scanning %} uses fingerprints to match results across various runs so they only appear once in the latest run for the selected branch. This makes it possible to match alerts to the correct line of code when files are edited. The `ruleId` for a result has to be the same across analysis. -### Reporting consistent filepaths +### Consistent filepaths The filepath has to be consistent across the runs to enable a computation of a stable fingerprint. If the filepaths differ for the same result, each time there is a new analysis a new alert will be created, and the old one will be closed. This will cause having multiple alerts for the same result. -### Including data for fingerprint generation +### Fingerprint generation {% data variables.product.prodname_dotcom %} uses the `partialFingerprints` property in the OASIS standard to detect when two results are logically identical. For more information, see the [partialFingerprints property](https://docs.oasis-open.org/sarif/sarif/v2.1.0/cs01/sarif-v2.1.0-cs01.html#_Toc16012611) entry in the OASIS documentation. @@ -50,17 +44,16 @@ SARIF files created by the {% data variables.code-scanning.codeql_workflow %}, o If you upload a SARIF file without fingerprint data using the `/code-scanning/sarifs` API endpoint, the {% data variables.product.prodname_code_scanning %} alerts will be processed and displayed, but users may see duplicate alerts. To avoid seeing duplicate alerts, you should calculate fingerprint data and populate the `partialFingerprints` property before you upload the SARIF file. You may find the script that the `upload-sarif` action uses a helpful starting point: https://github.com/github/codeql-action/blob/main/src/fingerprints.ts. For more information about the API, see [AUTOTITLE](/rest/code-scanning/code-scanning#upload-an-analysis-as-sarif-data). -## Understanding rules and results +## Rules and results SARIF files support both rules and results. The information stored in these elements is similar but serves different purposes. * Rules are an array of `reportingDescriptor` objects that are included in the `toolComponent` object. This is where you store details of the rules that are run during analysis. Information in these objects should change infrequently, typically when you update the tool. - * Results are stored as a series of `result` objects under `results` in the `run` object. Each `result` object contains details for one alert in the codebase. Within the `results` object, you can reference the rule that detected the alert. When you compare SARIF files generated by analyzing different codebases with the same tool and rules, you should see differences in the results of the analyses but not in the rules. -## Specifying the location for source files +## Source file locations Specifying source file locations and lines of code ensures code scanning alerts are displayed accurately within the file containing the identified problem, allowing for targeted issue resolution. @@ -75,7 +68,7 @@ For display in a pull request check, an alert must meet all the following condit The `physicalLocation` object in a submitted SARIF file identifies the lines of code for an alert. For more information, see [`physicalLocation` object](#physicallocation-object). -### Specifying the root for source files +### Root location for source files {% data variables.product.prodname_code_scanning_caps %} interprets results that are reported with relative paths as relative to the root of the repository analyzed. If a result contains an absolute URI, the URI is converted to a relative URI. The relative URI can then be matched against a file committed to the repository. @@ -98,35 +91,25 @@ file:///tmp/go-build/tmp.go -> file:///tmp/go-build/tmp.go The file is successfully uploaded as both absolute URIs use the same URI scheme as the source root. -### Resolving symlinks before generating SARIF results +### Symlink resolution If the relative URI for a result is matched against a file defined using a symlink, code scanning is unable to display the result. So you need to resolve any symlinked files, and report any results in those files using the resolved URI. -## Validating your SARIF file +## File compatibility You can check a SARIF file is compatible with {% data variables.product.prodname_code_scanning %} by testing it against the {% data variables.product.prodname_dotcom %} ingestion rules. For more information, visit the [Microsoft SARIF validator](https://sarifweb.azurewebsites.net/). -For each gzip-compressed SARIF file, SARIF upload supports a maximum size of 10 MB. Any uploads over this limit will be rejected. If your SARIF file is too large because it contains too many results, you should update the configuration to focus on results for the most important rules or queries. For more information, see [AUTOTITLE](/code-security/code-scanning/troubleshooting-sarif-uploads/file-too-large). +For each gzip-compressed SARIF file, SARIF upload supports a maximum size of 10 MB. Any uploads over this limit will be rejected. If your SARIF file is too large because it contains too many results, you should update the configuration to focus on results for the most important rules or queries. For more information, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/file-too-large). -{% data variables.product.prodname_code_scanning_caps %} supports uploading a maximum number of entries for the data objects in the following table. If any of these objects exceeds its maximum value the SARIF file is rejected. For some objects, there is also an additional limit on the number of values that will be displayed. Whenever possible the most important values are shown. To get the most out of your analysis when it includes data above the supported limits, try to optimize the analysis configuration (for example, for the {% data variables.product.prodname_codeql %} tool, identify and disable the most noisy queries). For more information, see [AUTOTITLE](/code-security/code-scanning/troubleshooting-sarif-uploads/results-exceed-limit). +{% data variables.product.prodname_code_scanning_caps %} supports uploading a maximum number of entries for the data objects in the following table. If any of these objects exceeds its maximum value the SARIF file is rejected. For some objects, there is also an additional limit on the number of values that will be displayed. Whenever possible the most important values are shown. To get the most out of your analysis when it includes data above the supported limits, try to optimize the analysis configuration (for example, for the {% data variables.product.prodname_codeql %} tool, identify and disable the most noisy queries). For more information, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads/results-exceed-limit). {% data reusables.code-scanning.sarif-limits %} -For information about other errors, see [AUTOTITLE](/code-security/code-scanning/troubleshooting-sarif-uploads) - -## Uploading more than one SARIF file for a commit - -You can upload multiple SARIF files for the same commit, and display the data from each file as {% data variables.product.prodname_code_scanning %} results. When you upload multiple SARIF files for a commit, you must indicate a "category" for each analysis. The way to specify a category varies according to the analysis method: -* Using the {% data variables.product.prodname_codeql_cli %} directly, pass the `--sarif-category` argument to the `codeql database analyze` command when you generate SARIF files. For more information, see [AUTOTITLE](/code-security/codeql-cli/getting-started-with-the-codeql-cli/about-the-codeql-cli#about-generating-code-scanning-results-with-the-codeql-cli). -* Using {% data variables.product.prodname_actions %} with `codeql-action/analyze`, the category is set automatically from the workflow name and any matrix variables (typically, `language`). You can override this by specifying a `category` input for the action, which is useful when you analyze different sections of a monorepo in a single workflow. -* Using {% data variables.product.prodname_actions %} to upload results from other static analysis tools, then you must specify a `category` input if you upload more than one file of results for the same tool in one workflow. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/uploading-a-sarif-file-to-github#uploading-a-code-scanning-analysis-with-github-actions). -* If you are not using either of these approaches, you must specify a unique `runAutomationDetails.id` in each SARIF file to upload. For more information about this property, see [`runAutomationDetails` object](#runautomationdetails-object). - -If you upload a second SARIF file for a commit with the same category and from the same tool, the earlier results are overwritten. However, if you try to upload multiple SARIF files for the same tool and category in a single {% data variables.product.prodname_actions %} workflow run, the misconfiguration is detected and the run will fail. +For information about other errors, see [AUTOTITLE](/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-sarif-uploads). -## Supported SARIF output file properties +## Supported properties If you use a code analysis engine other than {% data variables.product.prodname_codeql %}, you can review the supported SARIF properties to optimize how your analysis results will appear on {% data variables.product.prodname_dotcom %}. @@ -151,7 +134,7 @@ Any valid SARIF 2.1.0 output file can be uploaded, however, {% data variables.pr |----|----|----| | `tool.driver` | {% octicon "check" aria-label="Required" %} | A `toolComponent` object that describes the analysis tool. For more information, see the [`toolComponent` object](#toolcomponent-object). | | `tool.extensions[]` | {% octicon "x" aria-label="Optional" %} | An array of `toolComponent` objects that represent any plugins or extensions used by the tool during analysis. For more information, see the [`toolComponent` object](#toolcomponent-object). | -| `invocation.workingDirectory.uri` | {% octicon "x" aria-label="Optional" %} | This field is used only when `checkout_uri` (SARIF upload API only) or `checkout_path` ({% data variables.product.prodname_actions %} only) are not provided. The value is used to convert absolute URIs used in [`physicalLocation` objects](#physicallocation-object) to relative URIs. For more information, see [Specifying the root for source files](#specifying-the-root-for-source-files).| +| `invocation.workingDirectory.uri` | {% octicon "x" aria-label="Optional" %} | This field is used only when `checkout_uri` (SARIF upload API only) or `checkout_path` ({% data variables.product.prodname_actions %} only) are not provided. The value is used to convert absolute URIs used in [`physicalLocation` objects](#physicallocation-object) to relative URIs. For more information, see [Specifying the root for source files](#root-location-for-source-files).| | `results[]` | {% octicon "check" aria-label="Required" %} | The results of the analysis tool. {% data variables.product.prodname_code_scanning_caps %} displays the results on {% data variables.product.prodname_dotcom %}. For more information, see the [`result` object](#result-object). ### `toolComponent` object @@ -165,7 +148,7 @@ Any valid SARIF 2.1.0 output file can be uploaded, however, {% data variables.pr ### `reportingDescriptor` object -This is where you store details of the rules that are run during analysis. Information in these objects should change infrequently, typically when you update the tool. For more information, see [Understanding rules and results](#understanding-rules-and-results) above. +This is where you store details of the rules that are run during analysis. Information in these objects should change infrequently, typically when you update the tool. For more information, see [Rules and results](#rules-and-results) above. | Name | Required | Description | |----|----|----| @@ -183,7 +166,7 @@ This is where you store details of the rules that are run during analysis. Infor ### `result` object -Each `result` object contains details for one alert in the codebase. Within the `results` object, you can reference the rule that detected the alert. For more information, see [Understanding rules and results](#understanding-rules-and-results) above. +Each `result` object contains details for one alert in the codebase. Within the `results` object, you can reference the rule that detected the alert. For more information, see [Rules and results](#rules-and-results) above. {% data reusables.code-scanning.upload-sarif-alert-limit %} @@ -195,7 +178,7 @@ Each `result` object contains details for one alert in the codebase. Within the | `level`| {% octicon "x" aria-label="Optional" %} | The severity of the result. This level overrides the default severity defined by the rule. {% data variables.product.prodname_code_scanning_caps %} uses the level to filter results by severity on {% data variables.product.prodname_dotcom %}. | `message.text`| {% octicon "check" aria-label="Required" %} | A message that describes the result. {% data variables.product.prodname_code_scanning_caps %} displays the message text as the title of the result. Only the first sentence of the message will be displayed when visible space is limited. | `locations[]`| {% octicon "check" aria-label="Required" %} | The set of locations where the result was detected up to a maximum of 10. Only one location should be included unless the problem can only be corrected by making a change at every specified location. **Note:** At least one location is required for {% data variables.product.prodname_code_scanning %} to display a result. {% data variables.product.prodname_code_scanning_caps %} will use this property to decide which file to annotate with the result. Only the first value of this array is used. All other values are ignored. -| `partialFingerprints`| {% octicon "check" aria-label="Required" %} | A set of strings used to track the unique identity of the result. {% data variables.product.prodname_code_scanning_caps %} uses `partialFingerprints` to accurately identify which results are the same across commits and branches. {% data variables.product.prodname_code_scanning_caps %} will attempt to use `partialFingerprints` if they exist. If you are uploading third-party SARIF files with the `upload-action`, the action will create `partialFingerprints` for you when they are not included in the SARIF file. For more information, see [Providing data to track code scanning alerts across runs](#providing-data-to-track-code-scanning-alerts-across-runs). **Note:** {% data variables.product.prodname_code_scanning_caps %} only uses the `primaryLocationLineHash`. +| `partialFingerprints`| {% octicon "check" aria-label="Required" %} | A set of strings used to track the unique identity of the result. {% data variables.product.prodname_code_scanning_caps %} uses `partialFingerprints` to accurately identify which results are the same across commits and branches. {% data variables.product.prodname_code_scanning_caps %} will attempt to use `partialFingerprints` if they exist. If you are uploading third-party SARIF files with the `upload-action`, the action will create `partialFingerprints` for you when they are not included in the SARIF file. For more information, see [Providing data to track code scanning alerts across runs](#data-for-preventing-duplicated-alerts). **Note:** {% data variables.product.prodname_code_scanning_caps %} only uses the `primaryLocationLineHash`. | `codeFlows[].threadFlows[].locations[]`| {% octicon "x" aria-label="Optional" %} | An array of `location` objects for a `threadFlow` object, which describes the progress of a program through a thread of execution. A `codeFlow` object describes a pattern of code execution used to detect a result. If code flows are provided, {% data variables.product.prodname_code_scanning %} will expand code flows on {% data variables.product.prodname_dotcom %} for the relevant result. For more information, see the [`location` object](#location-object). | `relatedLocations[]`| {% octicon "x" aria-label="Optional" %} | A set of locations relevant to this result. {% data variables.product.prodname_code_scanning_caps %} will link to related locations when they are embedded in the result message. For more information, see the [`location` object](#location-object). @@ -213,7 +196,7 @@ A location within a programming artifact, such as a file in the repository or a | Name | Required | Description | |----|----|----| -| `artifactLocation.uri`| {% octicon "check" aria-label="Required" %} | A URI indicating the location of an artifact, usually a file either in the repository or generated during a build. For the best results we recommend that this is a relative path from the root of the GitHub repository being analyzed. For example, `src/main.js`. For more information about artifact URIs, see [Specifying the root for source files](#specifying-the-root-for-source-files).| +| `artifactLocation.uri`| {% octicon "check" aria-label="Required" %} | A URI indicating the location of an artifact, usually a file either in the repository or generated during a build. For the best results we recommend that this is a relative path from the root of the GitHub repository being analyzed. For example, `src/main.js`. For more information about artifact URIs, see [Specifying the root for source files](#root-location-for-source-files).| | `region.startLine` | {% octicon "check" aria-label="Required" %} | The line number of the first character in the region. | `region.startColumn` | {% octicon "check" aria-label="Required" %} | The column number of the first character in the region. | `region.endLine` | {% octicon "check" aria-label="Required" %} | The line number of the last character in the region. diff --git a/content/code-security/reference/secret-security/index.md b/content/code-security/reference/secret-security/index.md index 06e6024a26de..e1982eaf7cb8 100644 --- a/content/code-security/reference/secret-security/index.md +++ b/content/code-security/reference/secret-security/index.md @@ -13,4 +13,5 @@ children: - /understanding-github-secret-types - /supported-secret-scanning-patterns - /risk-report-csv-contents + - /secret-scanning-pattern-configuration-data --- diff --git a/content/code-security/reference/secret-security/secret-scanning-pattern-configuration-data.md b/content/code-security/reference/secret-security/secret-scanning-pattern-configuration-data.md new file mode 100644 index 000000000000..3d99e0889688 --- /dev/null +++ b/content/code-security/reference/secret-security/secret-scanning-pattern-configuration-data.md @@ -0,0 +1,23 @@ +--- +title: Secret scanning pattern configuration data +shortTitle: Secret pattern data +intro: Understand the data displayed in the {% data variables.product.prodname_secret_scanning %} pattern configuration page to make informed decisions about push protection settings. +permissions: '{% data reusables.permissions.secret-risk-assessment-report-generation %}' +versions: + feature: security-configurations +topics: + - Code Security + - Secret scanning + - Secret Protection + - Organizations + - Security + - Advanced Security + - Enterprise +contentType: reference + +--- + +When configuring push protection, you can view performance data for each secret pattern to make informed enablement decisions. Use metrics like alert volume and false positive rates to balance security with developer experience. See [AUTOTITLE](/code-security/how-tos/secure-at-scale/configure-organization-security/establish-complete-coverage/configuring-global-security-settings-for-your-organization#specifying-patterns-to-include-in-push-protection){% ifversion security-configuration-enterprise-level %} or [AUTOTITLE](/code-security/how-tos/secure-at-scale/configure-enterprise-security/establish-complete-coverage/configuring-additional-secret-scanning-settings-for-your-enterprise#specifying-patterns-to-include-in-push-protection-for-your-enterprise){% endif %}. + +{% data reusables.secret-scanning.pattern-enablement-org-enterprise-chart %} + \ No newline at end of file diff --git a/content/code-security/tutorials/customize-code-scanning/creating-and-working-with-codeql-packs.md b/content/code-security/tutorials/customize-code-scanning/creating-and-working-with-codeql-packs.md index 95083cf19990..db28c15daa57 100644 --- a/content/code-security/tutorials/customize-code-scanning/creating-and-working-with-codeql-packs.md +++ b/content/code-security/tutorials/customize-code-scanning/creating-and-working-with-codeql-packs.md @@ -21,40 +21,9 @@ contentType: tutorials {% data reusables.code-scanning.codeql-cli-version-ghes %} -{% data variables.product.prodname_codeql %} packs are used to create, share, depend on, and run {% data variables.product.prodname_codeql %} queries and libraries. {% data variables.product.prodname_codeql %} packs contain queries, library files, query suites, and metadata. With {% data variables.product.prodname_codeql %} packs and the package management commands in the {% data variables.product.prodname_codeql_cli %}, you can publish your custom queries and integrate them into your codebase analysis. +{% data variables.product.prodname_codeql %} packs are used to create, share, depend on, and run {% data variables.product.prodname_codeql %} queries and libraries. -There are three types of {% data variables.product.prodname_codeql %} packs: query packs, library packs, and model packs. - -* Query packs are designed to be run. When a query pack is published, the bundle includes all the transitive dependencies and pre-compiled representations of each query, in addition to the query sources. This ensures consistent and efficient execution of the queries in the pack. - -* Library packs are designed to be used by query packs (or other library packs) and do not contain queries themselves. The libraries are not compiled separately. - -* Model packs can be used to expand {% data variables.product.prodname_code_scanning %} analysis to include dependencies that are not supported by default. Model packs are currently in {% data variables.release-phases.public_preview %} and subject to change. During the {% data variables.release-phases.public_preview %}, model packs are available for {% data variables.code-scanning.codeql_model_packs_support %} analysis. For more information about creating your own model packs, see [Creating a {% data variables.product.prodname_codeql %} model pack](#creating-a-codeql-model-pack). - -You can use the `pack` command in the {% data variables.product.prodname_codeql_cli %} to create {% data variables.product.prodname_codeql %} packs, add dependencies to packs, and install or update dependencies. You can also publish and download {% data variables.product.prodname_codeql %} packs using the `pack` command. For more information, see [AUTOTITLE](/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/publishing-and-using-codeql-packs). - -For more information about compatibility between published query packs and different {% data variables.product.prodname_codeql %} releases, see [AUTOTITLE](/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/publishing-and-using-codeql-packs#about-codeql-pack-compatibility). - -The standard {% data variables.product.prodname_codeql %} packages for all supported languages are published in the [{% data variables.product.prodname_container_registry %}](https://github.com/orgs/codeql/packages). The [{% data variables.product.prodname_codeql %} repository](https://github.com/github/codeql) contains source files for the standard {% data variables.product.prodname_codeql %} packs for all supported languages. The core query packs, which are included in the {% data variables.product.prodname_codeql %} CLI bundle, but you can otherwise download, are: - - * `codeql/cpp-queries` - * `codeql/csharp-queries` - * `codeql/go-queries` - * `codeql/java-queries` - * `codeql/javascript-queries` - * `codeql/python-queries` - * `codeql/ruby-queries` - -## {% data variables.product.prodname_codeql %} pack structure - -A {% data variables.product.prodname_codeql %} pack must contain a file called `qlpack.yml` in its root directory. In the `qlpack.yml` file, the `name:` field must have a value that follows the format of `/`, where `` is the {% data variables.product.prodname_dotcom %} organization or user account that the pack will be published to and `` is the name of the pack. Additionally, query packs and library packs with {% data variables.product.prodname_codeql %} tests contain a `codeql-pack.lock.yml` file that contains the resolved dependencies of the pack. This file is generated during a call to the `codeql pack install` command, is not meant to be edited by hand, and should be added to your version control system. - -The other files and directories within the pack should be logically organized. For example, typically: - -* Queries are organized into directories for specific categories. - -* Queries for specific products, libraries, and frameworks are organized into -their own top-level directories. +You can use the `pack` command in the {% data variables.product.prodname_codeql_cli %} to create {% data variables.product.prodname_codeql %} packs, add dependencies to packs, and install or update dependencies. You can also publish and download {% data variables.product.prodname_codeql %} packs using the `pack` command. ## Creating a {% data variables.product.prodname_codeql %} pack @@ -128,6 +97,18 @@ This command downloads all dependencies to the shared cache on the local disk. > * Running the `codeql pack add` and `codeql pack install` commands will generate or update the `codeql-pack.lock.yml` file. This file should be checked-in to version control. The `codeql-pack.lock.yml` file contains the precise version numbers used by the pack. For more information, see [About codeql-pack.lock.yml files](/code-security/codeql-cli/getting-started-with-the-codeql-cli/customizing-analysis-with-codeql-packs##about-codeql-packlockyml-files). > * By default `codeql pack install` will install dependencies from the {% data variables.product.prodname_container_registry %} on {% data variables.product.prodname_dotcom_the_website %}. You can install dependencies from a {% data variables.product.prodname_ghe_server %} {% data variables.product.prodname_container_registry %} by creating a `qlconfig.yml` file. For more information, see [AUTOTITLE](/enterprise-server@latest/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/publishing-and-using-codeql-packs) in the {% data variables.product.prodname_ghe_server %} documentation. +## Organizing a {% data variables.product.prodname_codeql %} pack's directory structure + +A {% data variables.product.prodname_codeql %} pack must contain a file called `qlpack.yml` in its root directory. In the `qlpack.yml` file, the `name:` field must have a value that follows the format of `/`, where `` is the {% data variables.product.prodname_dotcom %} organization or user account that the pack will be published to and `` is the name of the pack. + +Additionally, query packs and library packs with {% data variables.product.prodname_codeql %} tests contain a `codeql-pack.lock.yml` file that contains the resolved dependencies of the pack. This file is generated during a call to the `codeql pack install` command, is not meant to be edited by hand, and should be added to your version control system. + +The other files and directories within the pack should be logically organized. For example, typically: + +* Queries are organized into directories for specific categories. +* Queries for specific products, libraries, and frameworks are organized into +their own top-level directories. + ## Customizing a downloaded {% data variables.product.prodname_codeql %} pack The recommended way to experiment with changes to a pack is to clone the repository containing its source code. diff --git a/content/enterprise-onboarding/getting-started-with-your-enterprise/index.md b/content/enterprise-onboarding/getting-started-with-your-enterprise/index.md index b3478fcdfd53..93a66ff1a5a9 100644 --- a/content/enterprise-onboarding/getting-started-with-your-enterprise/index.md +++ b/content/enterprise-onboarding/getting-started-with-your-enterprise/index.md @@ -15,5 +15,3 @@ children: - /about-enterprise-billing - /about-migrating-to-github-enterprise-cloud --- - -Visit [{% data variables.product.prodname_enterprise %}](https://github.com/enterprise) to discover how our AI-powered developer platform can benefit you, and to start a free trial of today. diff --git a/data/reusables/code-scanning/upload-sarif-alert-limit.md b/data/reusables/code-scanning/upload-sarif-alert-limit.md index 0691e32e0fdb..e8d843d2e552 100644 --- a/data/reusables/code-scanning/upload-sarif-alert-limit.md +++ b/data/reusables/code-scanning/upload-sarif-alert-limit.md @@ -1 +1 @@ -You can check that the SARIF properties have the supported size for upload and that the file is compatible with code scanning. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#validating-your-sarif-file). +You can check that the SARIF properties have the supported size for upload and that the file is compatible with code scanning. For more information, see [AUTOTITLE](/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#file-compatibility). diff --git a/data/reusables/secret-scanning/pattern-enablement-org-enterprise-chart.md b/data/reusables/secret-scanning/pattern-enablement-org-enterprise-chart.md new file mode 100644 index 000000000000..cf167044e4d7 --- /dev/null +++ b/data/reusables/secret-scanning/pattern-enablement-org-enterprise-chart.md @@ -0,0 +1,9 @@ + | Column | Description | + |---------------------------------------------|----------------------------------------------| + | Name | Name of the pattern or secret | + | Alert total | Total number of alerts for the pattern (percentage and absolute numbers) | + | False positives | Percentage of false positives for the pattern | + | Bypass rate | Percentage of bypasses for the pattern | + | {% data variables.product.github %} default | Default behavior for push protection, as recommended by {% data variables.product.github %} | + | Enterprise setting | **Uneditable at organization level**
Current enablement status for push protection
Can be `Enabled`, `Disabled`, and `Default`.
At enterprise level, `Default` is the default value. | + | Organization setting | **Only valid at organization level**
Current enablement status for push protection
Can be `Enabled`, `Disabled`, and `Enterprise` (inherited from the enterprise).
`Enterprise` is the default value. | diff --git a/data/reusables/secret-scanning/pattern-enablement-org-enterprise.md b/data/reusables/secret-scanning/pattern-enablement-org-enterprise.md index abd26ff7c8d8..8fec37fa124d 100644 --- a/data/reusables/secret-scanning/pattern-enablement-org-enterprise.md +++ b/data/reusables/secret-scanning/pattern-enablement-org-enterprise.md @@ -6,12 +6,3 @@ > [!NOTE] Organization administrators and security teams can override settings configured at the enterprise level. - | Column | Description | - |---------------------------------------------|----------------------------------------------| - | Name | Name of the pattern or secret | - | Alert total | Total number of alerts for the pattern (percentage and absolute numbers) | - | False positives | Percentage of false positives for the pattern | - | Bypass rate | Percentage of bypasses for the pattern | - | {% data variables.product.github %} default | Default behavior for push protection, as recommended by {% data variables.product.github %} | - | Enterprise setting | **Uneditable at organization level**
Current enablement status for push protection
Can be `Enabled`, `Disabled`, and `Default`.
At enterprise level, `Default` is the default value. | - | Organization setting | **Only valid at organization level**
Current enablement status for push protection
Can be `Enabled`, `Disabled`, and `Enterprise` (inherited from the enterprise).
`Enterprise` is the default value. | diff --git a/src/article-api/README.md b/src/article-api/README.md index 98667b88e554..68ad97d2e00a 100644 --- a/src/article-api/README.md +++ b/src/article-api/README.md @@ -1,7 +1,10 @@ # Article API This subject folder contains the code for the Article API endpoints: + - `/api/pagelist` +- `/api/pagelist/versions` +- `/api/pagelist/languages` - `/api/article/body` - `/api/article/meta` @@ -17,7 +20,7 @@ The `/api/article/body` endpoint can serve markdown for both regular articles an ## How it works -The `/api/article` endpoints return information about a page by `pathname`. +The `/api/article` endpoints return information about a page by `pathname`. `api/article/meta` is highly cached, in JSON format. @@ -28,14 +31,15 @@ For autogenerated pages (REST, GraphQL, webhooks, landing pages, audit logs, etc #### Transformers To add a new transformer for other autogenerated content types: + 1. Create a new transformer file implementing the `PageTransformer` interface 2. Register it in `transformers/index.ts` -3. Create a template in `templates/` to configure how the transformer will organize the autogenerated content +3. Create a template in `templates/` to configure how the transformer will organize the autogenerated content 4. The transformer will automatically be used by `/api/article/body` ## How to get help -For internal folks ask in the Docs Engineering slack channel. +For internal folks ask in the Docs Engineering slack channel. For open source folks, please open a discussion in the public repository. @@ -139,6 +143,42 @@ Get metadata about an article. --- +### GET /api/pagelist/versions + +Get all available product versions for the docs site. + +**Returns**: (object) - JSON object with version information + +**Example**: +``` +❯ curl -s https://docs.github.com/api/pagelist/versions | jq +{ + "versions": ["free-pro-team@latest", "enterprise-cloud@latest", "enterprise-server@3.19", ...], + "ghesVersions": ["3.19", "3.18", "3.17", ...], + "ghesLatest": "3.19", + ... +} +``` + +--- + +### GET /api/pagelist/languages + +Get all available languages for the docs site. + +**Returns**: (object) - JSON object with language information + +**Example**: +``` +❯ curl -s https://docs.github.com/api/pagelist/languages | jq +{ + "languages": ["en", "es", "ja", "pt", "zh", "ru", "fr", "ko", "de"], + "allLanguages": { ... } +} +``` + +--- + ### GET /api/pagelist/:lang/:productVersion A list of pages available for a fully qualified path containing the target language and product version. @@ -165,34 +205,3 @@ A list of pages available for a fully qualified path containing the target langu --- - -## Local Development - -### Bootstrap steps - -1. Start the dev server: - ```bash - npm run dev - ``` - -2. Test API endpoints locally: - ```bash - curl http://localhost:4000/api/article/meta?pathname=/en/get-started - curl http://localhost:4000/api/article/body?pathname=/en/get-started - curl http://localhost:4000/api/pagelist/en/free-pro-team@latest - ``` - -### Running tests - -```bash -npm run test -- src/article-api/tests -``` - -## Ownership - -- Team: Docs Engineering - -### Known limitations -- Cache invalidation is manual -- No built-in rate limiting (uses Fastly instead) -- Limited API versioning diff --git a/src/article-api/middleware/pagelist.ts b/src/article-api/middleware/pagelist.ts index 3b69a032b5f7..095f4b9bab59 100644 --- a/src/article-api/middleware/pagelist.ts +++ b/src/article-api/middleware/pagelist.ts @@ -8,11 +8,85 @@ import { getLanguageCodeFromPath } from '@/languages/middleware/detect-language' import { pagelistValidationMiddleware } from './validation' import catchMiddlewareError from '@/observability/middleware/catch-middleware-error' import statsd from '@/observability/lib/statsd' +import { allVersions, allVersionKeys } from '@/versions/lib/all-versions' +import enterpriseServerReleases from '@/versions/lib/enterprise-server-releases' +import { languages, languageKeys } from '@/languages/lib/languages' const router = express.Router() // pagelistValidationMiddleware is used for every route to normalize the lang and version from the path +/** + * Get all available product versions for the docs site. + * @route GET /api/pagelist/versions + * @returns {object} JSON object with version information + * @example + * ❯ curl -s https://docs.github.com/api/pagelist/versions | jq + * { + * "versions": ["free-pro-team@latest", "enterprise-cloud@latest", "enterprise-server@3.19", ...], + * "ghesVersions": ["3.19", "3.18", "3.17", ...], + * "ghesLatest": "3.19", + * ... + * } + */ +router.get( + '/versions', + catchMiddlewareError(async function (req: ExtendedRequest, res: Response) { + defaultCacheControl(res) + + const response = { + // Simple list of all version strings + versions: allVersionKeys, + // GHES-specific information + ghesVersions: enterpriseServerReleases.supported, + ghesLatest: enterpriseServerReleases.latest, + ghesLatestStable: enterpriseServerReleases.latestStable, + ghesReleaseCandidate: enterpriseServerReleases.releaseCandidate, + ghesDeprecated: enterpriseServerReleases.deprecated, + // Full version details + allVersions, + } + + res.json(response) + }) as RequestHandler, +) + +/** + * Get all available languages for the docs site. + * @route GET /api/pagelist/languages + * @returns {object} JSON object with language information + * @example + * ❯ curl -s https://docs.github.com/api/pagelist/languages | jq + * { + * "languages": ["en", "es", "ja", "pt", "zh", "ru", "fr", "ko", "de"], + * "allLanguages": { ... } + * } + */ +router.get( + '/languages', + catchMiddlewareError(async function (req: ExtendedRequest, res: Response) { + defaultCacheControl(res) + + // Remove redirectPatterns from output as they are RegExp objects and not JSON serializable + const sanitizedLanguages = Object.fromEntries( + Object.entries(languages).map(([code, lang]) => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { redirectPatterns, ...rest } = lang + return [code, rest] + }), + ) + + const response = { + // Simple list of language codes + languages: languageKeys, + // Full language details (without redirectPatterns) + allLanguages: sanitizedLanguages, + } + + res.json(response) + }) as RequestHandler, +) + // If no version or lang is provided we'll assume english and fpt and redirect there router.get( '/', diff --git a/src/article-api/tests/pagelist.ts b/src/article-api/tests/pagelist.ts index 8c22fa29c276..0cdb55cfcd04 100644 --- a/src/article-api/tests/pagelist.ts +++ b/src/article-api/tests/pagelist.ts @@ -73,3 +73,71 @@ describe('Redirect Tests', () => { expect(res.headers.location).toBe('/api/pagelist/en/free-pro-team@latest') }) }) + +describe('Versions API', () => { + test('returns 200 and JSON with version information', async () => { + const res = await get('/api/pagelist/versions') + expect(res.statusCode).toBe(200) + expect(res.headers['content-type']).toMatch(/application\/json/) + + const data = JSON.parse(res.body) + + // Check top-level keys exist + expect(data).toHaveProperty('versions') + expect(data).toHaveProperty('ghesVersions') + expect(data).toHaveProperty('ghesLatest') + expect(data).toHaveProperty('ghesLatestStable') + expect(data).toHaveProperty('ghesReleaseCandidate') + expect(data).toHaveProperty('ghesDeprecated') + expect(data).toHaveProperty('allVersions') + + // Versions array should contain expected values + expect(Array.isArray(data.versions)).toBe(true) + expect(data.versions).toContain('free-pro-team@latest') + expect(data.versions).toContain('enterprise-cloud@latest') + + // GHES versions should be an array of version strings + expect(Array.isArray(data.ghesVersions)).toBe(true) + expect(data.ghesVersions.length).toBeGreaterThan(0) + expect(data.ghesVersions[0]).toMatch(/^\d+\.\d+$/) + + // ghesLatest should be a valid version string + expect(data.ghesLatest).toMatch(/^\d+\.\d+$/) + + // allVersions should be an object with version details + expect(typeof data.allVersions).toBe('object') + expect(data.allVersions['free-pro-team@latest']).toHaveProperty('version') + expect(data.allVersions['free-pro-team@latest']).toHaveProperty('versionTitle') + }) +}) + +describe('Languages API', () => { + test('returns 200 and JSON with language information', async () => { + const res = await get('/api/pagelist/languages') + expect(res.statusCode).toBe(200) + expect(res.headers['content-type']).toMatch(/application\/json/) + + const data = JSON.parse(res.body) + + // Check top-level keys exist + expect(data).toHaveProperty('languages') + expect(data).toHaveProperty('allLanguages') + + // Languages array should contain expected values + expect(Array.isArray(data.languages)).toBe(true) + expect(data.languages).toContain('en') + expect(data.languages).toContain('ja') + expect(data.languages).toContain('es') + + // allLanguages should be an object with language details + expect(typeof data.allLanguages).toBe('object') + expect(data.allLanguages.en).toHaveProperty('name') + expect(data.allLanguages.en).toHaveProperty('code') + expect(data.allLanguages.en).toHaveProperty('hreflang') + expect(data.allLanguages.en).toHaveProperty('locale') + expect(data.allLanguages.en.name).toBe('English') + + // Should not contain redirectPatterns (not JSON serializable) + expect(data.allLanguages.ja).not.toHaveProperty('redirectPatterns') + }) +}) diff --git a/src/frame/middleware/llms-txt.ts b/src/frame/middleware/llms-txt.ts index 88020d11ecb9..1ae9ea740d6b 100644 --- a/src/frame/middleware/llms-txt.ts +++ b/src/frame/middleware/llms-txt.ts @@ -56,6 +56,8 @@ function generateBasicLlmsTxt(): string { ## Docs Content - [Page List API](${BASE_API_URL}/en/free-pro-team@latest) +- [Versions API](${BASE_API_URL}/versions): \`curl "https://docs.github.com/api/pagelist/versions"\` +- [Languages API](${BASE_API_URL}/languages): \`curl "https://docs.github.com/api/pagelist/languages"\` - [Article API](https://docs.github.com/api/article): \`curl "https://docs.github.com/api/article?pathname=/en/get-started/start-your-journey/about-github-and-git"\` - [Search API](https://docs.github.com/api/search): \`curl "https://docs.github.com/api/search?query=actions&language=en&version=free-pro-team@latest"\`