diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift')
128 files changed, 25865 insertions, 0 deletions
diff --git a/compiler/rustc_codegen_cranelift/.cirrus.yml b/compiler/rustc_codegen_cranelift/.cirrus.yml new file mode 100644 index 00000000000..5a464bfac36 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.cirrus.yml @@ -0,0 +1,20 @@ +task: + name: freebsd + freebsd_instance: + image: freebsd-13-2-release-amd64 + setup_rust_script: + - pkg install -y git bash binutils + - curl https://sh.rustup.rs -sSf --output rustup.sh + - sh rustup.sh --default-toolchain none -y --profile=minimal + target_cache: + folder: build/cg_clif + prepare_script: + - . $HOME/.cargo/env + - ./y.sh prepare + test_script: + - . $HOME/.cargo/env + # Disabling incr comp reduces cache size and incr comp doesn't save as much + # on CI anyway. + - export CARGO_BUILD_INCREMENTAL=false + # Skip rand as it fails on FreeBSD due to rust-random/rand#1355 + - ./y.sh test --skip-test test.rust-random/rand diff --git a/compiler/rustc_codegen_cranelift/.gitattributes b/compiler/rustc_codegen_cranelift/.gitattributes new file mode 100644 index 00000000000..0ceb3fe646c --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.gitattributes @@ -0,0 +1,2 @@ +* text=auto eol=lf +*.rs diff=rust diff --git a/compiler/rustc_codegen_cranelift/.github/actions/github-release/README.md b/compiler/rustc_codegen_cranelift/.github/actions/github-release/README.md new file mode 100644 index 00000000000..c70ba8f4953 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/actions/github-release/README.md @@ -0,0 +1,18 @@ +# github-release + +An action used to publish GitHub releases for `wasmtime`. + +As of the time of this writing there's a few actions floating around which +perform github releases but they all tend to have their set of drawbacks. +Additionally nothing handles deleting releases which we need for our rolling +`dev` release. + +To handle all this this action rolls-its-own implementation using the +actions/toolkit repository and packages published there. These run in a Docker +container and take various inputs to orchestrate the release from the build. + +More comments can be found in `main.js`. + +Testing this is really hard. If you want to try though run `npm install` and +then `node main.js`. You'll have to configure a bunch of env vars though to get +anything reasonably working. diff --git a/compiler/rustc_codegen_cranelift/.github/actions/github-release/action.yml b/compiler/rustc_codegen_cranelift/.github/actions/github-release/action.yml new file mode 100644 index 00000000000..36e5209f50c --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/actions/github-release/action.yml @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +name: 'rustc_codegen_cranelift github releases' +description: 'rustc_codegen_cranelift github releases' +inputs: + token: + description: '' + required: true + files: + description: '' + required: true +runs: + using: 'node16' + main: 'main.js' diff --git a/compiler/rustc_codegen_cranelift/.github/actions/github-release/main.js b/compiler/rustc_codegen_cranelift/.github/actions/github-release/main.js new file mode 100644 index 00000000000..1eb2b7f23b2 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/actions/github-release/main.js @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const core = require('@actions/core'); +const path = require("path"); +const fs = require("fs"); +const github = require('@actions/github'); +const glob = require('glob'); + +function sleep(milliseconds) { + return new Promise(resolve => setTimeout(resolve, milliseconds)) +} + +async function runOnce() { + // Load all our inputs and env vars. Note that `getInput` reads from `INPUT_*` + const files = core.getInput('files'); + const token = core.getInput('token'); + const slug = process.env.GITHUB_REPOSITORY; + const owner = slug.split('/')[0]; + const repo = slug.split('/')[1]; + const sha = process.env.GITHUB_SHA; + let name = 'dev'; + if (process.env.GITHUB_REF.startsWith('refs/tags/v')) { + name = process.env.GITHUB_REF.substring(10); + } + + core.info(`files: ${files}`); + core.info(`name: ${name}`); + core.info(`token: ${token}`); + + const octokit = github.getOctokit(token); + + // For the `dev` release we may need to update the tag to point to the new + // commit on this branch. All other names should already have tags associated + // with them. + if (name == 'dev') { + let tag = null; + try { + tag = await octokit.request("GET /repos/:owner/:repo/git/refs/tags/:name", { owner, repo, name }); + core.info(`found existing tag`); + console.log("tag: ", JSON.stringify(tag.data, null, 2)); + } catch (e) { + // ignore if this tag doesn't exist + core.info(`no existing tag found`); + } + + if (tag === null || tag.data.object.sha !== sha) { + core.info(`updating existing tag or creating new one`); + + try { + core.info(`updating dev tag`); + await octokit.rest.git.updateRef({ + owner, + repo, + ref: 'tags/dev', + sha, + force: true, + }); + } catch (e) { + console.log("ERROR: ", JSON.stringify(e.response, null, 2)); + core.info(`creating dev tag`); + try { + await octokit.rest.git.createRef({ + owner, + repo, + ref: 'refs/tags/dev', + sha, + }); + } catch (e) { + // we might race with others, so assume someone else has created the + // tag by this point. + console.log("failed to create tag: ", JSON.stringify(e.response, null, 2)); + } + } + + console.log("double-checking tag is correct"); + tag = await octokit.request("GET /repos/:owner/:repo/git/refs/tags/:name", { owner, repo, name }); + if (tag.data.object.sha !== sha) { + console.log("tag: ", JSON.stringify(tag.data, null, 2)); + throw new Error("tag didn't work"); + } + } else { + core.info(`existing tag works`); + } + } + + // Delete a previous release + try { + core.info(`fetching release`); + let release = await octokit.rest.repos.getReleaseByTag({ owner, repo, tag: name }); + console.log("found release: ", JSON.stringify(release.data, null, 2)); + await octokit.rest.repos.deleteRelease({ + owner, + repo, + release_id: release.data.id, + }); + console.log("deleted release"); + } catch (e) { + console.log("ERROR: ", JSON.stringify(e, null, 2)); + } + + console.log("creating a release"); + let release = await octokit.rest.repos.createRelease({ + owner, + repo, + tag_name: name, + prerelease: name === 'dev', + }); + + // Delete all assets from a previous run + for (const asset of release.data.assets) { + console.log(`deleting prior asset ${asset.id}`); + await octokit.rest.repos.deleteReleaseAsset({ + owner, + repo, + asset_id: asset.id, + }); + } + + // Upload all the relevant assets for this release as just general blobs. + for (const file of glob.sync(files)) { + const size = fs.statSync(file).size; + const name = path.basename(file); + core.info(`upload ${file}`); + await octokit.rest.repos.uploadReleaseAsset({ + data: fs.createReadStream(file), + headers: { 'content-length': size, 'content-type': 'application/octet-stream' }, + name, + url: release.data.upload_url, + }); + } +} + +async function run() { + const retries = 10; + for (let i = 0; i < retries; i++) { + try { + await runOnce(); + break; + } catch (e) { + if (i === retries - 1) + throw e; + logError(e); + console.log("RETRYING after 10s"); + await sleep(10000) + } + } +} + +function logError(e) { + console.log("ERROR: ", e.message); + try { + console.log(JSON.stringify(e, null, 2)); + } catch (e) { + // ignore json errors for now + } + console.log(e.stack); +} + +run().catch(err => { + logError(err); + core.setFailed(err.message); +}); diff --git a/compiler/rustc_codegen_cranelift/.github/actions/github-release/package-lock.json b/compiler/rustc_codegen_cranelift/.github/actions/github-release/package-lock.json new file mode 100644 index 00000000000..dd3b2a048f0 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/actions/github-release/package-lock.json @@ -0,0 +1,571 @@ +{ + "name": "rustc_codegen_cranelift-github-release", + "version": "0.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "rustc_codegen_cranelift-github-release", + "version": "0.0.0", + "dependencies": { + "@actions/core": "^1.9.1", + "@actions/github": "^5.1.0", + "glob": "^7.1.5" + } + }, + "node_modules/@actions/core": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", + "dependencies": { + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" + } + }, + "node_modules/@actions/github": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-5.1.0.tgz", + "integrity": "sha512-tuI80F7JQIhg77ZTTgUAPpVD7ZnP9oHSPN8xw7LOwtA4vEMbAjWJNbmLBfV7xua7r016GyjzWLuec5cs8f/a8A==", + "dependencies": { + "@actions/http-client": "^2.0.1", + "@octokit/core": "^3.6.0", + "@octokit/plugin-paginate-rest": "^2.17.0", + "@octokit/plugin-rest-endpoint-methods": "^5.13.0" + } + }, + "node_modules/@actions/http-client": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", + "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", + "dependencies": { + "tunnel": "^0.0.6" + } + }, + "node_modules/@octokit/auth-token": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", + "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", + "dependencies": { + "@octokit/types": "^6.0.3" + } + }, + "node_modules/@octokit/core": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz", + "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", + "dependencies": { + "@octokit/auth-token": "^2.4.4", + "@octokit/graphql": "^4.5.8", + "@octokit/request": "^5.6.3", + "@octokit/request-error": "^2.0.5", + "@octokit/types": "^6.0.3", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@octokit/endpoint": { + "version": "6.0.12", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", + "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", + "dependencies": { + "@octokit/types": "^6.0.3", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@octokit/graphql": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", + "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", + "dependencies": { + "@octokit/request": "^5.6.0", + "@octokit/types": "^6.0.3", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "12.11.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", + "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==" + }, + "node_modules/@octokit/plugin-paginate-rest": { + "version": "2.21.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz", + "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==", + "dependencies": { + "@octokit/types": "^6.40.0" + }, + "peerDependencies": { + "@octokit/core": ">=2" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz", + "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==", + "dependencies": { + "@octokit/types": "^6.39.0", + "deprecation": "^2.3.1" + }, + "peerDependencies": { + "@octokit/core": ">=3" + } + }, + "node_modules/@octokit/request": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz", + "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", + "dependencies": { + "@octokit/endpoint": "^6.0.1", + "@octokit/request-error": "^2.1.0", + "@octokit/types": "^6.16.1", + "is-plain-object": "^5.0.0", + "node-fetch": "^2.6.7", + "universal-user-agent": "^6.0.0" + } + }, + "node_modules/@octokit/request-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", + "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", + "dependencies": { + "@octokit/types": "^6.0.3", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "node_modules/@octokit/types": { + "version": "6.41.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", + "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", + "dependencies": { + "@octokit/openapi-types": "^12.11.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "node_modules/deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, + "node_modules/universal-user-agent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", + "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + } + }, + "dependencies": { + "@actions/core": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", + "requires": { + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" + } + }, + "@actions/github": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-5.1.0.tgz", + "integrity": "sha512-tuI80F7JQIhg77ZTTgUAPpVD7ZnP9oHSPN8xw7LOwtA4vEMbAjWJNbmLBfV7xua7r016GyjzWLuec5cs8f/a8A==", + "requires": { + "@actions/http-client": "^2.0.1", + "@octokit/core": "^3.6.0", + "@octokit/plugin-paginate-rest": "^2.17.0", + "@octokit/plugin-rest-endpoint-methods": "^5.13.0" + } + }, + "@actions/http-client": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", + "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", + "requires": { + "tunnel": "^0.0.6" + } + }, + "@octokit/auth-token": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", + "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", + "requires": { + "@octokit/types": "^6.0.3" + } + }, + "@octokit/core": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz", + "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", + "requires": { + "@octokit/auth-token": "^2.4.4", + "@octokit/graphql": "^4.5.8", + "@octokit/request": "^5.6.3", + "@octokit/request-error": "^2.0.5", + "@octokit/types": "^6.0.3", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/endpoint": { + "version": "6.0.12", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", + "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", + "requires": { + "@octokit/types": "^6.0.3", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", + "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", + "requires": { + "@octokit/request": "^5.6.0", + "@octokit/types": "^6.0.3", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/openapi-types": { + "version": "12.11.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", + "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==" + }, + "@octokit/plugin-paginate-rest": { + "version": "2.21.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz", + "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==", + "requires": { + "@octokit/types": "^6.40.0" + } + }, + "@octokit/plugin-rest-endpoint-methods": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz", + "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==", + "requires": { + "@octokit/types": "^6.39.0", + "deprecation": "^2.3.1" + } + }, + "@octokit/request": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz", + "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", + "requires": { + "@octokit/endpoint": "^6.0.1", + "@octokit/request-error": "^2.1.0", + "@octokit/types": "^6.16.1", + "is-plain-object": "^5.0.0", + "node-fetch": "^2.6.7", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/request-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", + "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", + "requires": { + "@octokit/types": "^6.0.3", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "6.41.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", + "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", + "requires": { + "@octokit/openapi-types": "^12.11.0" + } + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" + }, + "universal-user-agent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", + "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + } + } +} diff --git a/compiler/rustc_codegen_cranelift/.github/actions/github-release/package.json b/compiler/rustc_codegen_cranelift/.github/actions/github-release/package.json new file mode 100644 index 00000000000..d9c23f8873e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/actions/github-release/package.json @@ -0,0 +1,11 @@ +{ + "name": "rustc_codegen_cranelift-github-release", + "version": "0.0.0", + "license": "Apache-2.0 WITH LLVM-exception", + "main": "main.js", + "dependencies": { + "@actions/core": "^1.9.1", + "@actions/github": "^5.1.0", + "glob": "^7.1.5" + } +} diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml b/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml new file mode 100644 index 00000000000..30dc5cb1615 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml @@ -0,0 +1,72 @@ +name: Abi-cafe + +on: + - push + - pull_request + +permissions: {} + +jobs: + abi_cafe: + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}-${{ matrix.env.TARGET_TRIPLE }} + cancel-in-progress: true + + defaults: + run: + shell: bash + + strategy: + fail-fast: true + matrix: + include: + - os: ubuntu-latest + env: + TARGET_TRIPLE: x86_64-unknown-linux-gnu + - os: macos-latest + env: + TARGET_TRIPLE: x86_64-apple-darwin + - os: macos-latest + env: + TARGET_TRIPLE: aarch64-apple-darwin + - os: windows-latest + env: + TARGET_TRIPLE: x86_64-pc-windows-msvc + # FIXME Currently hangs. Re-enable once this is fixed or abi-cafe adds a timeout. + #- os: windows-latest + # env: + # TARGET_TRIPLE: x86_64-pc-windows-gnu + + steps: + - uses: actions/checkout@v4 + + - name: CPU features + if: matrix.os == 'ubuntu-latest' + run: cat /proc/cpuinfo + + - name: Cache cargo target dir + uses: actions/cache@v4 + with: + path: build/cg_clif + key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + + - name: Set MinGW as the default toolchain + if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + run: rustup set default-host x86_64-pc-windows-gnu + + - name: Use x86_64 compiler on macOS + if: matrix.os == 'macos-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-apple-darwin' + run: rustup set default-host x86_64-apple-darwin + + - name: Prepare dependencies + run: ./y.sh prepare + + - name: Build + run: ./y.sh build --sysroot none + + - name: Test abi-cafe + env: + TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} + run: ./y.sh abi-cafe diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/audit.yml b/compiler/rustc_codegen_cranelift/.github/workflows/audit.yml new file mode 100644 index 00000000000..27c95572ef8 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/workflows/audit.yml @@ -0,0 +1,18 @@ +name: Security audit +on: + workflow_dispatch: + schedule: + - cron: '0 10 * * 1' # every monday at 10:00 UTC +permissions: + issues: write + checks: write +jobs: + audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + sed -i 's/components.*/components = []/' rust-toolchain + - uses: rustsec/audit-check@v1.4.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml new file mode 100644 index 00000000000..896a5c34c3e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml @@ -0,0 +1,320 @@ +name: CI + +on: + - push + - pull_request + +defaults: + run: + shell: bash + +permissions: {} + +env: + # Disabling incr comp reduces cache size and incr comp doesn't save as much + # on CI anyway. + CARGO_BUILD_INCREMENTAL: false + # Rust's CI denies warnings. Deny them here too to ensure subtree syncs don't + # fail because of warnings. + RUSTFLAGS: "-Dwarnings" + +jobs: + rustfmt: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Avoid installing rustc-dev + run: | + sed -i 's/components.*/components = ["rustfmt"]/' rust-toolchain + rustfmt -v + + - name: Rustfmt + run: | + cargo fmt --check + rustfmt --check build_system/main.rs + rustfmt --check example/* + rustfmt --check scripts/*.rs + + + test: + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + + env: + CG_CLIF_EXPENSIVE_CHECKS: 1 + + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + env: + TARGET_TRIPLE: x86_64-unknown-linux-gnu + - os: macos-latest + env: + TARGET_TRIPLE: x86_64-apple-darwin + # cross-compile from Linux to Windows using mingw + - os: ubuntu-latest + env: + TARGET_TRIPLE: x86_64-pc-windows-gnu + apt_deps: gcc-mingw-w64-x86-64 wine-stable + - os: ubuntu-latest + env: + TARGET_TRIPLE: aarch64-unknown-linux-gnu + apt_deps: gcc-aarch64-linux-gnu qemu-user + - os: macos-latest + env: + TARGET_TRIPLE: aarch64-apple-darwin + - os: ubuntu-latest + env: + TARGET_TRIPLE: s390x-unknown-linux-gnu + apt_deps: gcc-s390x-linux-gnu qemu-user + - os: ubuntu-latest + env: + TARGET_TRIPLE: riscv64gc-unknown-linux-gnu + apt_deps: gcc-riscv64-linux-gnu qemu-user + - os: windows-latest + env: + TARGET_TRIPLE: x86_64-pc-windows-msvc + - os: windows-latest + env: + TARGET_TRIPLE: x86_64-pc-windows-gnu + + steps: + - uses: actions/checkout@v4 + + - name: CPU features + if: matrix.os == 'ubuntu-latest' + run: cat /proc/cpuinfo + + - name: Cache cargo target dir + uses: actions/cache@v4 + with: + path: build/cg_clif + key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + + - name: Set MinGW as the default toolchain + if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + run: rustup set default-host x86_64-pc-windows-gnu + + - name: Use x86_64 compiler on macOS + if: matrix.os == 'macos-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-apple-darwin' + run: rustup set default-host x86_64-apple-darwin + + - name: Install toolchain and emulator + if: matrix.apt_deps != null + run: | + sudo apt-get update + sudo apt-get install -y ${{ matrix.apt_deps }} + + - name: Prepare dependencies + run: ./y.sh prepare + + # The Wine version shipped with Ubuntu 22.04 doesn't implement bcryptprimitives.dll + - name: Build bcryptprimitives.dll shim for Wine + if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + run: | + rustup target add x86_64-pc-windows-gnu + mkdir wine_shims + rustc patches/bcryptprimitives.rs -Copt-level=3 -Clto=fat --out-dir wine_shims --target x86_64-pc-windows-gnu + echo "WINEPATH=$(pwd)/wine_shims" >> $GITHUB_ENV + + - name: Build + run: ./y.sh build --sysroot none + + - name: Test + env: + TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} + run: ./y.sh test + + - name: Install LLVM standard library + run: rustup target add ${{ matrix.env.TARGET_TRIPLE }} + + # This is roughly config rust-lang/rust uses for testing + - name: Test with LLVM sysroot + # Skip native x86_64-pc-windows-gnu. It is way too slow and cross-compiled + # x86_64-pc-windows-gnu covers at least part of the tests. + if: matrix.os != 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu' + env: + TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} + run: ./y.sh test --sysroot llvm --no-unstable-features + + + # This job doesn't use cg_clif in any way. It checks that all cg_clif tests work with cg_llvm too. + test_llvm: + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + + - name: CPU features + run: cat /proc/cpuinfo + + - name: Prepare dependencies + run: ./y.sh prepare + + - name: Disable JIT tests + run: | + sed -i 's/jit./#jit./' config.txt + + - name: Test + env: + TARGET_TRIPLE: x86_64-unknown-linux-gnu + run: ./y.sh test --use-backend llvm + + bench: + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + + - name: CPU features + run: cat /proc/cpuinfo + + - name: Cache cargo target dir + uses: actions/cache@v4 + with: + path: build/cg_clif + key: ${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + + - name: Install hyperfine + run: | + sudo apt update + sudo apt install -y hyperfine + + - name: Prepare dependencies + run: ./y.sh prepare + + - name: Build + run: ./y.sh build --sysroot none + + - name: Benchmark + run: ./y.sh bench + + + dist: + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + include: + # FIXME update at some point in the future once most distros use a newer glibc + - os: ubuntu-20.04 + env: + TARGET_TRIPLE: x86_64-unknown-linux-gnu + - os: macos-latest + env: + TARGET_TRIPLE: x86_64-apple-darwin + - os: macos-latest + env: + TARGET_TRIPLE: aarch64-apple-darwin + # cross-compile from Linux to Windows using mingw + - os: ubuntu-latest + env: + TARGET_TRIPLE: x86_64-pc-windows-gnu + - os: windows-latest + env: + TARGET_TRIPLE: x86_64-pc-windows-msvc + - os: windows-latest + env: + TARGET_TRIPLE: x86_64-pc-windows-gnu + + steps: + - uses: actions/checkout@v4 + + - name: Cache cargo target dir + uses: actions/cache@v4 + with: + path: build/cg_clif + key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-dist-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + + - name: Set MinGW as the default toolchain + if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + run: rustup set default-host x86_64-pc-windows-gnu + + - name: Use x86_64 compiler on macOS + if: matrix.os == 'macos-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-apple-darwin' + run: rustup set default-host x86_64-apple-darwin + + - name: Install MinGW toolchain + if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + run: | + sudo apt-get update + sudo apt-get install -y gcc-mingw-w64-x86-64 + + - name: Prepare dependencies + run: ./y.sh prepare + + - name: Build backend + run: ./y.sh build --sysroot none + + - name: Build sysroot + run: ./y.sh build + + - name: Package prebuilt cg_clif + run: tar cvfJ cg_clif.tar.xz dist + + - name: Upload prebuilt cg_clif + if: matrix.os == 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu' + uses: actions/upload-artifact@v4 + with: + name: cg_clif-${{ matrix.env.TARGET_TRIPLE }} + path: cg_clif.tar.xz + + - name: Upload prebuilt cg_clif (cross compile) + if: matrix.os != 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + uses: actions/upload-artifact@v4 + with: + name: cg_clif-${{ runner.os }}-cross-x86_64-mingw + path: cg_clif.tar.xz + + release: + runs-on: ubuntu-latest + timeout-minutes: 10 + if: ${{ github.ref == 'refs/heads/master' }} + # FIXME add the bench job back to the dependency list once rust-lang/rust#125493 gets merged + needs: [rustfmt, test, dist] + + permissions: + contents: write # for creating the dev tag and release + + concurrency: + group: release-dev + cancel-in-progress: true + + steps: + - uses: actions/checkout@v4 + + - name: Download all built artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts/ + + - run: | + ls -R artifacts/ + mkdir release/ + pushd artifacts/ + for dir in *; do + mv $dir/cg_clif.tar.xz ../release/$dir.tar.xz + rmdir $dir/ # verify $dir is empty + done + popd + rmdir artifacts/ # verify all artifacts are represented in release/ + ls -R release/ + + - run: npm install --production + working-directory: .github/actions/github-release + + - name: Publish Release + uses: ./.github/actions/github-release + with: + files: "release/*" + token: ${{ github.token }} + continue-on-error: true diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml new file mode 100644 index 00000000000..70c214ce8b1 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml @@ -0,0 +1,57 @@ +name: Various rustc tests + +on: + - push + +permissions: {} + +jobs: + bootstrap_rustc: + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + + - name: CPU features + run: cat /proc/cpuinfo + + - name: Cache cargo target dir + uses: actions/cache@v4 + with: + path: build/cg_clif + key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain', 'Cargo.lock') }} + + - name: Prepare dependencies + run: ./y.sh prepare + + - name: Test + run: ./scripts/test_bootstrap.sh + + + rustc_test_suite: + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + + - name: CPU features + run: cat /proc/cpuinfo + + - name: Cache cargo target dir + uses: actions/cache@v4 + with: + path: build/cg_clif + key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain', 'Cargo.lock') }} + + - name: Install ripgrep + run: | + sudo apt update + sudo apt install -y ripgrep + + - name: Prepare dependencies + run: ./y.sh prepare + + - name: Test + run: ./scripts/test_rustc_tests.sh diff --git a/compiler/rustc_codegen_cranelift/.gitignore b/compiler/rustc_codegen_cranelift/.gitignore new file mode 100644 index 00000000000..5a38f2acb0e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.gitignore @@ -0,0 +1,17 @@ +# Build artifacts during normal use +/download +/build +/dist +/target +/build_system/target + +# Downloaded by certain scripts +/rust +/git-fixed-subtree.sh + +# Various things that can be created during development +*.rlib +*.o +perf.data +perf.data.old +*.mm_profdata diff --git a/compiler/rustc_codegen_cranelift/.vscode/settings.json b/compiler/rustc_codegen_cranelift/.vscode/settings.json new file mode 100644 index 00000000000..491646ce59b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.vscode/settings.json @@ -0,0 +1,48 @@ +{ + "editor.formatOnSave": true, + + // in case rustc.source is disabled for performance reasons; disable the errors about this + "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"], + "rust-analyzer.rustc.source": "discover", + "rust-analyzer.imports.granularity.enforce": true, + "rust-analyzer.imports.granularity.group": "module", + "rust-analyzer.imports.prefix": "crate", + "rust-analyzer.cargo.features": ["unstable-features"], + "rust-analyzer.linkedProjects": [ + "./Cargo.toml", + "./build_system/Cargo.toml", + { + "crates": [ + { + "root_module": "./example/mini_core.rs", + "edition": "2018", + "deps": [], + "cfg": [], + }, + { + "root_module": "./example/mini_core_hello_world.rs", + "edition": "2018", + "deps": [{ "crate": 0, "name": "mini_core" }], + "cfg": [], + }, + { + "root_module": "./example/mod_bench.rs", + "edition": "2018", + "deps": [], + "cfg": [], + }, + ] + }, + { + "sysroot_src": "./build/stdlib/library", + "crates": [ + { + "root_module": "./example/std_example.rs", + "edition": "2015", + "deps": [], + "cfg": [], + }, + ] + } + ] +} diff --git a/compiler/rustc_codegen_cranelift/.zed/settings.json b/compiler/rustc_codegen_cranelift/.zed/settings.json new file mode 100644 index 00000000000..e93bed36949 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.zed/settings.json @@ -0,0 +1,68 @@ +{ + "format_on_save": "on", + "lsp": { + "rust-analyzer": { + "initialization_options": { + "diagnostics": { + // in case rustc.source is disabled for performance reasons; disable the errors about this + "disabled": ["unresolved-extern-crate", "unresolved-macro-call"] + }, + "rustc": { + "source": "discover" + }, + "imports": { + "granularity": { + "enforce": true, + "group": "module" + }, + "prefix": "crate" + }, + "cargo": { + "features": ["unstable-features"] + }, + "linkedProjects": [ + "./Cargo.toml", + "./build_system/Cargo.toml", + { + "crates": [ + { + "root_module": "./example/mini_core.rs", + "edition": "2018", + "deps": [], + "cfg": [] + }, + { + "root_module": "./example/mini_core_hello_world.rs", + "edition": "2018", + "deps": [ + { + "crate": 0, + "name": "mini_core" + } + ], + "cfg": [] + }, + { + "root_module": "./example/mod_bench.rs", + "edition": "2018", + "deps": [], + "cfg": [] + } + ] + }, + { + "sysroot_src": "./build/stdlib/library", + "crates": [ + { + "root_module": "./example/std_example.rs", + "edition": "2015", + "deps": [], + "cfg": [] + } + ] + } + ] + } + } + } +} diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock new file mode 100644 index 00000000000..741a1748fec --- /dev/null +++ b/compiler/rustc_codegen_cranelift/Cargo.lock @@ -0,0 +1,557 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "ar_archive_writer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2bcb7cf51decfbbfc7ef476e28b0775b13e5eb1190f8b7df145cd53d4f4374" +dependencies = [ + "object", +] + +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cranelift-bforest" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effa84ab2023f7138045ece6b326588c17447ca22e66db71ec15cb0a6c0c4ad2" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-bitset" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a1dfc50dca188a15d938867c4400589530bcb0138f7022aae6d059d1d8c309" + +[[package]] +name = "cranelift-codegen" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "821c20c639350158ecca928dc2a244d0d1c9cef2377a378fc62a445a286eb1ca" +dependencies = [ + "bumpalo", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli", + "hashbrown 0.14.5", + "log", + "regalloc2", + "rustc-hash", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064473f2fd59b44fa2c9aaa60de1f9c44db5e13521e28bc85d2b92ee535ef625" +dependencies = [ + "cranelift-codegen-shared", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f39b9ebfd2febdc2acfb9a0fca110665bcd5a6839502576307735ed07b2177" + +[[package]] +name = "cranelift-control" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e125c189c3a1ca8dfe209fc6f46edba058a6d24e0b92aff69459a15f4711e7" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea62eb109baec2247e1a6fa7b74c0f584b1e76e289cfd7017385b4b031fc8450" +dependencies = [ + "cranelift-bitset", +] + +[[package]] +name = "cranelift-frontend" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "722b089357aacb6c7528b2e59a5fe00917d61ce63448b25a3e477a5b7819fac8" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4b5005a48288e7fc2a2991a377831c534e26929b063c379c018060727785a9b" + +[[package]] +name = "cranelift-jit" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f843932baf8d1025c5f114b929eda172d74b7163d058e0de2597c308b567c7e9" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-module", + "cranelift-native", + "libc", + "log", + "region", + "target-lexicon", + "wasmtime-jit-icache-coherence", + "windows-sys", +] + +[[package]] +name = "cranelift-module" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449819ef1c4af139cf1b9717916fcaea0e23248853d3e95135139773a842d3eb" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-control", +] + +[[package]] +name = "cranelift-native" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae2d48f38081a9e679ad795bd36bb29bedeb5552fc1c195185bf9885fa1b16e" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-object" +version = "0.110.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a39ee2cfd0ec485eca76f6b4dc17701a280fa406bc05137bb43f1635ed12c9f" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-control", + "cranelift-module", + "log", + "object", + "target-lexicon", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +dependencies = [ + "fallible-iterator", + "indexmap", + "stable_deref_trait", +] + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", +] + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libloading" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "object" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" +dependencies = [ + "crc32fast", + "hashbrown 0.14.5", + "indexmap", + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regalloc2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad156d539c879b7a24a363a2016d77961786e71f48f2e2fc8302a92abd2429a6" +dependencies = [ + "hashbrown 0.13.2", + "log", + "rustc-hash", + "slice-group-by", + "smallvec", +] + +[[package]] +name = "region" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0" +dependencies = [ + "bitflags", + "libc", + "mach", + "winapi", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_codegen_cranelift" +version = "0.1.0" +dependencies = [ + "ar_archive_writer", + "cranelift-codegen", + "cranelift-frontend", + "cranelift-jit", + "cranelift-module", + "cranelift-native", + "cranelift-object", + "gimli", + "indexmap", + "libloading", + "object", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "slice-group-by" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "syn" +version = "2.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasmtime-jit-icache-coherence" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fddf3e2980fb1d123d1fcac55189e417fdd3dba4f62139b5a0a1f9efe5669d5" +dependencies = [ + "anyhow", + "cfg-if", + "libc", + "windows-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml new file mode 100644 index 00000000000..c86919dcdfb --- /dev/null +++ b/compiler/rustc_codegen_cranelift/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "rustc_codegen_cranelift" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["dylib"] + +[dependencies] +# These have to be in sync with each other +cranelift-codegen = { version = "0.110.1", default-features = false, features = ["std", "unwind", "all-arch"] } +cranelift-frontend = { version = "0.110.1" } +cranelift-module = { version = "0.110.1" } +cranelift-native = { version = "0.110.1" } +cranelift-jit = { version = "0.110.1", optional = true } +cranelift-object = { version = "0.110.1" } +target-lexicon = "0.12.0" +gimli = { version = "0.28", default-features = false, features = ["write"]} +object = { version = "0.36", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } +ar_archive_writer = "0.3" + +indexmap = "2.0.0" +libloading = { version = "0.8.0", optional = true } +smallvec = "1.8.1" + +[patch.crates-io] +# Uncomment to use local checkout of cranelift +#cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } +#cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } +#cranelift-module = { path = "../wasmtime/cranelift/module" } +#cranelift-native = { path = "../wasmtime/cranelift/native" } +#cranelift-jit = { path = "../wasmtime/cranelift/jit" } +#cranelift-object = { path = "../wasmtime/cranelift/object" } + +#gimli = { path = "../" } + +[features] +# Enable features not ready to be enabled when compiling as part of rustc +unstable-features = ["jit", "inline_asm_sym"] +jit = ["cranelift-jit", "libloading"] +inline_asm_sym = [] + +[package.metadata.rust-analyzer] +rustc_private = true diff --git a/compiler/rustc_codegen_cranelift/LICENSE-APACHE b/compiler/rustc_codegen_cranelift/LICENSE-APACHE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/compiler/rustc_codegen_cranelift/LICENSE-MIT b/compiler/rustc_codegen_cranelift/LICENSE-MIT new file mode 100644 index 00000000000..31aa79387f2 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md new file mode 100644 index 00000000000..6766e2f844d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/Readme.md @@ -0,0 +1,124 @@ +# Cranelift codegen backend for rust + +The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/main/cranelift). +This has the potential to improve compilation times in debug mode. +If your project doesn't use any of the things listed under "Not yet supported", it should work fine. +If not please open an issue. + +## Download using Rustup + +The Cranelift codegen backend is distributed in nightly builds on Linux and x86_64 macOS. If you want to +install it using Rustup, you can do that by running: + +```bash +$ rustup component add rustc-codegen-cranelift-preview --toolchain nightly +``` + +Once it is installed, you can enable it with one of the following approaches: +- `CARGO_PROFILE_DEV_CODEGEN_BACKEND=cranelift cargo +nightly build -Zcodegen-backend` +- Add the following to `.cargo/config.toml`: + ```toml + [unstable] + codegen-backend = true + + [profile.dev] + codegen-backend = "cranelift" + ``` +- Add the following to `Cargo.toml`: + ```toml + # This line needs to come before anything else in Cargo.toml + cargo-features = ["codegen-backend"] + + [profile.dev] + codegen-backend = "cranelift" + ``` + +## Precompiled builds + +You can also download a pre-built version from the [releases] page. +Extract the `dist` directory in the archive anywhere you want. +If you want to use `cargo clif build` instead of having to specify the full path to the `cargo-clif` executable, you can add the `bin` subdirectory of the extracted `dist` directory to your `PATH`. +(tutorial [for Windows](https://stackoverflow.com/a/44272417), and [for Linux/MacOS](https://unix.stackexchange.com/questions/26047/how-to-correctly-add-a-path-to-path/26059#26059)). + +[releases]: https://github.com/rust-lang/rustc_codegen_cranelift/releases/tag/dev + +## Building and testing + +If you want to build the backend manually, you can download it from GitHub and build it yourself: + +```bash +$ git clone https://github.com/rust-lang/rustc_codegen_cranelift +$ cd rustc_codegen_cranelift +$ ./y.sh prepare +$ ./y.sh build +``` + +To run the test suite replace the last command with: + +```bash +$ ./test.sh +``` + +For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.sh`. + +## Platform support + +|OS \ architecture|x86\_64|AArch64|Riscv64|s390x (System-Z)| +|---|---|---|---|---| +|Linux|✅|✅|✅[^no-rustup]|✅[^no-rustup]| +|FreeBSD|✅[^no-rustup]|❓|❓|❓| +|AIX|❌[^xcoff]|N/A|N/A|❌[^xcoff]| +|Other unixes|❓|❓|❓|❓| +|macOS|✅|✅|N/A|N/A| +|Windows|✅[^no-rustup]|❌|N/A|N/A| + +✅: Fully supported and tested +❓: Maybe supported, not tested +❌: Not supported at all + +Not all targets are available as rustup component for nightly. See notes in the platform support matrix. + +[^xcoff]: XCOFF object file format is not supported. +[^no-rustup]: Not available as rustup component for nightly. You can build it yourself. + +## Usage + +rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. + +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.sh prepare` and `y.sh build` or `test.sh`). + +In the directory with your project (where you can do the usual `cargo build`), run: + +```bash +$ $cg_clif_dir/dist/cargo-clif build +``` + +This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend. + +For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md). + +## Building and testing with changes in rustc code + +See [rustc_testing.md](docs/rustc_testing.md). + +## Not yet supported + +* SIMD ([tracked here](https://github.com/rust-lang/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported) +* Unwinding on panics ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1677), `-Cpanic=abort` is enabled by default) + +## License + +Licensed under either of + + * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you shall be dual licensed as above, without any +additional terms or conditions. diff --git a/compiler/rustc_codegen_cranelift/build_system/Cargo.lock b/compiler/rustc_codegen_cranelift/build_system/Cargo.lock new file mode 100644 index 00000000000..86268e19160 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "y" +version = "0.1.0" diff --git a/compiler/rustc_codegen_cranelift/build_system/Cargo.toml b/compiler/rustc_codegen_cranelift/build_system/Cargo.toml new file mode 100644 index 00000000000..feed2b6eafe --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "y" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "y" +path = "main.rs" + +[features] +unstable-features = [] # for rust-analyzer + +# Do not add any dependencies + +[profile.dev] +debug = 1 diff --git a/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs b/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs new file mode 100644 index 00000000000..e3f1162445b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs @@ -0,0 +1,72 @@ +use crate::path::{Dirs, RelPath}; +use crate::prepare::GitRepo; +use crate::utils::{spawn_and_wait, CargoProject, Compiler}; +use crate::{build_sysroot, CodegenBackend, SysrootKind}; + +static ABI_CAFE_REPO: GitRepo = GitRepo::github( + "Gankra", + "abi-cafe", + "f1220cfd13b57f5c0082c26529163865ee25e115", + "fe93a9acd461425d", + "abi-cafe", +); + +static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe_target"); + +pub(crate) fn run( + channel: &str, + sysroot_kind: SysrootKind, + dirs: &Dirs, + cg_clif_dylib: &CodegenBackend, + rustup_toolchain_name: Option<&str>, + bootstrap_host_compiler: &Compiler, +) { + RelPath::DOWNLOAD.ensure_exists(dirs); + ABI_CAFE_REPO.fetch(dirs); + ABI_CAFE_REPO.patch(dirs); + + eprintln!("Building sysroot for abi-cafe"); + build_sysroot::build_sysroot( + dirs, + channel, + sysroot_kind, + cg_clif_dylib, + bootstrap_host_compiler, + rustup_toolchain_name, + bootstrap_host_compiler.triple.clone(), + ); + + eprintln!("Running abi-cafe"); + + let pairs = ["rustc_calls_cgclif", "cgclif_calls_rustc", "cgclif_calls_cc", "cc_calls_cgclif"]; + let pairs = + if cfg!(not(any(target_os = "macos", all(target_os = "windows", target_env = "msvc")))) { + &pairs[..] + } else { + &pairs[..2] + }; + + let mut cmd = ABI_CAFE.run(bootstrap_host_compiler, dirs); + cmd.arg("--"); + + // stdcall, vectorcall and such don't work yet + cmd.arg("--conventions").arg("c").arg("--conventions").arg("rust"); + + for pair in pairs { + cmd.arg("--pairs").arg(pair); + } + + cmd.arg("--add-rustc-codegen-backend"); + match cg_clif_dylib { + CodegenBackend::Local(path) => { + cmd.arg(format!("cgclif:{}", path.display())); + } + CodegenBackend::Builtin(name) => { + cmd.arg(format!("cgclif:{name}")); + } + } + + cmd.current_dir(ABI_CAFE.source_dir(dirs)); + + spawn_and_wait(cmd); +} diff --git a/compiler/rustc_codegen_cranelift/build_system/bench.rs b/compiler/rustc_codegen_cranelift/build_system/bench.rs new file mode 100644 index 00000000000..6c64faaa256 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/bench.rs @@ -0,0 +1,130 @@ +use std::env; +use std::io::Write; +use std::path::Path; + +use crate::path::{Dirs, RelPath}; +use crate::prepare::GitRepo; +use crate::rustc_info::get_file_name; +use crate::utils::{hyperfine_command, spawn_and_wait, Compiler}; + +static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github( + "ebobby", + "simple-raytracer", + "804a7a21b9e673a482797aa289a18ed480e4d813", + "ad6f59a2331a3f56", + "<none>", +); + +pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { + benchmark_simple_raytracer(dirs, bootstrap_host_compiler); +} + +fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { + if std::process::Command::new("hyperfine").output().is_err() { + eprintln!("Hyperfine not installed"); + eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine"); + std::process::exit(1); + } + + SIMPLE_RAYTRACER_REPO.fetch(dirs); + SIMPLE_RAYTRACER_REPO.patch(dirs); + + let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap(); + + let mut gha_step_summary = if let Ok(file) = std::env::var("GITHUB_STEP_SUMMARY") { + Some(std::fs::OpenOptions::new().append(true).open(file).unwrap()) + } else { + None + }; + + eprintln!("[BENCH COMPILE] ebobby/simple-raytracer"); + let cargo_clif = RelPath::DIST + .to_path(dirs) + .join(get_file_name(&bootstrap_host_compiler.rustc, "cargo_clif", "bin").replace('_', "-")); + let manifest_path = SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).join("Cargo.toml"); + let target_dir = RelPath::BUILD.join("simple_raytracer").to_path(dirs); + + let clean_cmd = format!( + "RUSTC=rustc cargo clean --manifest-path {manifest_path} --target-dir {target_dir}", + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + let llvm_build_cmd = format!( + "RUSTC=rustc cargo build --manifest-path {manifest_path} --target-dir {target_dir} && (rm build/raytracer_cg_llvm || true) && ln build/simple_raytracer/debug/main build/raytracer_cg_llvm", + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + let clif_build_cmd = format!( + "RUSTC=rustc {cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir} && (rm build/raytracer_cg_clif || true) && ln build/simple_raytracer/debug/main build/raytracer_cg_clif", + cargo_clif = cargo_clif.display(), + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + let clif_build_opt_cmd = format!( + "RUSTC=rustc {cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir} --release && (rm build/raytracer_cg_clif_opt || true) && ln build/simple_raytracer/release/main build/raytracer_cg_clif_opt", + cargo_clif = cargo_clif.display(), + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + + let bench_compile_markdown = RelPath::DIST.to_path(dirs).join("bench_compile.md"); + + let bench_compile = hyperfine_command( + 1, + bench_runs, + Some(&clean_cmd), + &[ + ("cargo build", &llvm_build_cmd), + ("cargo-clif build", &clif_build_cmd), + ("cargo-clif build --release", &clif_build_opt_cmd), + ], + &bench_compile_markdown, + ); + + spawn_and_wait(bench_compile); + + if let Some(gha_step_summary) = gha_step_summary.as_mut() { + gha_step_summary.write_all(b"## Compile ebobby/simple-raytracer\n\n").unwrap(); + gha_step_summary.write_all(&std::fs::read(bench_compile_markdown).unwrap()).unwrap(); + gha_step_summary.write_all(b"\n").unwrap(); + } + + eprintln!("[BENCH RUN] ebobby/simple-raytracer"); + + let bench_run_markdown = RelPath::DIST.to_path(dirs).join("bench_run.md"); + + let raytracer_cg_llvm = Path::new(".").join(get_file_name( + &bootstrap_host_compiler.rustc, + "raytracer_cg_llvm", + "bin", + )); + let raytracer_cg_clif = Path::new(".").join(get_file_name( + &bootstrap_host_compiler.rustc, + "raytracer_cg_clif", + "bin", + )); + let raytracer_cg_clif_opt = Path::new(".").join(get_file_name( + &bootstrap_host_compiler.rustc, + "raytracer_cg_clif_opt", + "bin", + )); + let mut bench_run = hyperfine_command( + 0, + bench_runs, + None, + &[ + ("", raytracer_cg_llvm.to_str().unwrap()), + ("", raytracer_cg_clif.to_str().unwrap()), + ("", raytracer_cg_clif_opt.to_str().unwrap()), + ], + &bench_run_markdown, + ); + bench_run.current_dir(RelPath::BUILD.to_path(dirs)); + spawn_and_wait(bench_run); + + if let Some(gha_step_summary) = gha_step_summary.as_mut() { + gha_step_summary.write_all(b"## Run ebobby/simple-raytracer\n\n").unwrap(); + gha_step_summary.write_all(&std::fs::read(bench_run_markdown).unwrap()).unwrap(); + gha_step_summary.write_all(b"\n").unwrap(); + } +} diff --git a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs new file mode 100644 index 00000000000..129713e574a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs @@ -0,0 +1,53 @@ +use std::env; +use std::path::PathBuf; + +use crate::path::{Dirs, RelPath}; +use crate::rustc_info::get_file_name; +use crate::shared_utils::{rustflags_from_env, rustflags_to_cmd_env}; +use crate::utils::{CargoProject, Compiler, LogGroup}; + +pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif"); + +pub(crate) fn build_backend( + dirs: &Dirs, + channel: &str, + bootstrap_host_compiler: &Compiler, + use_unstable_features: bool, +) -> PathBuf { + let _group = LogGroup::guard("Build backend"); + + let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs); + + let mut rustflags = rustflags_from_env("RUSTFLAGS"); + + rustflags.push("-Zallow-features=rustc_private".to_owned()); + + if env::var("CG_CLIF_EXPENSIVE_CHECKS").is_ok() { + // Enabling debug assertions implicitly enables the clif ir verifier + cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true"); + cmd.env("CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS", "true"); + } + + if use_unstable_features { + cmd.arg("--features").arg("unstable-features"); + } + + match channel { + "debug" => {} + "release" => { + cmd.arg("--release"); + } + _ => unreachable!(), + } + + rustflags_to_cmd_env(&mut cmd, "RUSTFLAGS", &rustflags); + + eprintln!("[BUILD] rustc_codegen_cranelift"); + crate::utils::spawn_and_wait(cmd); + + CG_CLIF + .target_dir(dirs) + .join(&bootstrap_host_compiler.triple) + .join(channel) + .join(get_file_name(&bootstrap_host_compiler.rustc, "rustc_codegen_cranelift", "dylib")) +} diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs new file mode 100644 index 00000000000..e41f6c5e709 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs @@ -0,0 +1,326 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{env, fs}; + +use crate::path::{Dirs, RelPath}; +use crate::rustc_info::get_file_name; +use crate::utils::{ + remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, LogGroup, +}; +use crate::{config, CodegenBackend, SysrootKind}; + +static DIST_DIR: RelPath = RelPath::DIST; +static BIN_DIR: RelPath = RelPath::DIST.join("bin"); +static LIB_DIR: RelPath = RelPath::DIST.join("lib"); + +pub(crate) fn build_sysroot( + dirs: &Dirs, + channel: &str, + sysroot_kind: SysrootKind, + cg_clif_dylib_src: &CodegenBackend, + bootstrap_host_compiler: &Compiler, + rustup_toolchain_name: Option<&str>, + target_triple: String, +) -> Compiler { + let _guard = LogGroup::guard("Build sysroot"); + + eprintln!("[BUILD] sysroot {:?}", sysroot_kind); + + DIST_DIR.ensure_fresh(dirs); + BIN_DIR.ensure_exists(dirs); + LIB_DIR.ensure_exists(dirs); + + let is_native = bootstrap_host_compiler.triple == target_triple; + + let cg_clif_dylib_path = match cg_clif_dylib_src { + CodegenBackend::Local(src_path) => { + // Copy the backend + let cg_clif_dylib_path = if cfg!(windows) { + // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the + // binaries. + BIN_DIR + } else { + LIB_DIR + } + .to_path(dirs) + .join(src_path.file_name().unwrap()); + try_hard_link(src_path, &cg_clif_dylib_path); + CodegenBackend::Local(cg_clif_dylib_path) + } + CodegenBackend::Builtin(name) => CodegenBackend::Builtin(name.clone()), + }; + + // Build and copy rustc and cargo wrappers + let wrapper_base_name = get_file_name(&bootstrap_host_compiler.rustc, "____", "bin"); + for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] { + let wrapper_name = wrapper_base_name.replace("____", wrapper); + + let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc); + let wrapper_path = DIST_DIR.to_path(dirs).join(&wrapper_name); + build_cargo_wrapper_cmd + .arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs"))) + .arg("-o") + .arg(&wrapper_path) + .arg("-Cstrip=debuginfo"); + if let Some(rustup_toolchain_name) = &rustup_toolchain_name { + build_cargo_wrapper_cmd + .env("TOOLCHAIN_NAME", rustup_toolchain_name) + .env_remove("CARGO") + .env_remove("RUSTC") + .env_remove("RUSTDOC"); + } else { + build_cargo_wrapper_cmd + .env_remove("TOOLCHAIN_NAME") + .env("CARGO", &bootstrap_host_compiler.cargo) + .env("RUSTC", &bootstrap_host_compiler.rustc) + .env("RUSTDOC", &bootstrap_host_compiler.rustdoc); + } + if let CodegenBackend::Builtin(name) = cg_clif_dylib_src { + build_cargo_wrapper_cmd.env("BUILTIN_BACKEND", name); + } + spawn_and_wait(build_cargo_wrapper_cmd); + try_hard_link(wrapper_path, BIN_DIR.to_path(dirs).join(wrapper_name)); + } + + let host = build_sysroot_for_triple( + dirs, + channel, + bootstrap_host_compiler.clone(), + &cg_clif_dylib_path, + sysroot_kind, + ); + host.install_into_sysroot(&DIST_DIR.to_path(dirs)); + + if !is_native { + build_sysroot_for_triple( + dirs, + channel, + { + let mut bootstrap_target_compiler = bootstrap_host_compiler.clone(); + bootstrap_target_compiler.triple = target_triple.clone(); + bootstrap_target_compiler.set_cross_linker_and_runner(); + bootstrap_target_compiler + }, + &cg_clif_dylib_path, + sysroot_kind, + ) + .install_into_sysroot(&DIST_DIR.to_path(dirs)); + } + + // Copy std for the host to the lib dir. This is necessary for the jit mode to find + // libstd. + for lib in host.libs { + let filename = lib.file_name().unwrap().to_str().unwrap(); + if filename.contains("std-") && !filename.contains(".rlib") { + try_hard_link(&lib, LIB_DIR.to_path(dirs).join(lib.file_name().unwrap())); + } + } + + let mut target_compiler = { + let dirs: &Dirs = &dirs; + let rustc_clif = + RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustc-clif")); + let rustdoc_clif = + RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustdoc-clif")); + + Compiler { + cargo: bootstrap_host_compiler.cargo.clone(), + rustc: rustc_clif.clone(), + rustdoc: rustdoc_clif.clone(), + rustflags: vec![], + rustdocflags: vec![], + triple: target_triple, + runner: vec![], + } + }; + if !is_native { + target_compiler.set_cross_linker_and_runner(); + } + target_compiler +} + +struct SysrootTarget { + triple: String, + libs: Vec<PathBuf>, +} + +impl SysrootTarget { + fn install_into_sysroot(&self, sysroot: &Path) { + if self.libs.is_empty() { + return; + } + + let target_rustlib_lib = sysroot.join("lib").join("rustlib").join(&self.triple).join("lib"); + fs::create_dir_all(&target_rustlib_lib).unwrap(); + + for lib in &self.libs { + try_hard_link(lib, target_rustlib_lib.join(lib.file_name().unwrap())); + } + } +} + +pub(crate) static STDLIB_SRC: RelPath = RelPath::BUILD.join("stdlib"); +pub(crate) static STANDARD_LIBRARY: CargoProject = + CargoProject::new(&STDLIB_SRC.join("library/sysroot"), "stdlib_target"); +pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup"); + +#[must_use] +fn build_sysroot_for_triple( + dirs: &Dirs, + channel: &str, + compiler: Compiler, + cg_clif_dylib_path: &CodegenBackend, + sysroot_kind: SysrootKind, +) -> SysrootTarget { + match sysroot_kind { + SysrootKind::None => build_rtstartup(dirs, &compiler) + .unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }), + SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler), + SysrootKind::Clif => { + build_clif_sysroot_for_triple(dirs, channel, compiler, cg_clif_dylib_path) + } + } +} + +#[must_use] +fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget { + let default_sysroot = crate::rustc_info::get_default_sysroot(&compiler.rustc); + + let mut target_libs = SysrootTarget { triple: compiler.triple, libs: vec![] }; + + for entry in fs::read_dir( + default_sysroot.join("lib").join("rustlib").join(&target_libs.triple).join("lib"), + ) + .unwrap() + { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + continue; + } + let file = entry.path(); + let file_name_str = file.file_name().unwrap().to_str().unwrap(); + if (file_name_str.contains("rustc_") + && !file_name_str.contains("rustc_std_workspace_") + && !file_name_str.contains("rustc_demangle")) + || file_name_str.contains("chalk") + || file_name_str.contains("tracing") + || file_name_str.contains("regex") + { + // These are large crates that are part of the rustc-dev component and are not + // necessary to run regular programs. + continue; + } + target_libs.libs.push(file); + } + + target_libs +} + +#[must_use] +fn build_clif_sysroot_for_triple( + dirs: &Dirs, + channel: &str, + mut compiler: Compiler, + cg_clif_dylib_path: &CodegenBackend, +) -> SysrootTarget { + let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; + + if let Some(rtstartup_target_libs) = build_rtstartup(dirs, &compiler) { + rtstartup_target_libs.install_into_sysroot(&RTSTARTUP_SYSROOT.to_path(dirs)); + + target_libs.libs.extend(rtstartup_target_libs.libs); + } + + let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join(channel); + + if !config::get_bool("keep_sysroot") { + // Cleanup the deps dir, but keep build scripts and the incremental cache for faster + // recompilation as they are not affected by changes in cg_clif. + remove_dir_if_exists(&build_dir.join("deps")); + } + + // Build sysroot + let mut rustflags = vec!["-Zforce-unstable-if-unmarked".to_owned(), "-Cpanic=abort".to_owned()]; + match cg_clif_dylib_path { + CodegenBackend::Local(path) => { + rustflags.push(format!("-Zcodegen-backend={}", path.to_str().unwrap())); + } + CodegenBackend::Builtin(name) => { + rustflags.push(format!("-Zcodegen-backend={name}")); + } + }; + // Necessary for MinGW to find rsbegin.o and rsend.o + rustflags.push("--sysroot".to_owned()); + rustflags.push(RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap().to_owned()); + if channel == "release" { + // Incremental compilation by default disables mir inlining. This leads to both a decent + // compile perf and a significant runtime perf regression. As such forcefully enable mir + // inlining. + rustflags.push("-Zinline-mir".to_owned()); + } + if let Some(prefix) = env::var_os("CG_CLIF_STDLIB_REMAP_PATH_PREFIX") { + rustflags.push("--remap-path-prefix".to_owned()); + rustflags.push(format!( + "{}={}", + STDLIB_SRC.to_path(dirs).to_str().unwrap(), + prefix.to_str().unwrap() + )); + } + compiler.rustflags.extend(rustflags); + let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); + if channel == "release" { + build_cmd.arg("--release"); + } + build_cmd.arg("--features").arg("backtrace panic-unwind compiler-builtins-no-f16-f128"); + build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true"); + build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); + if compiler.triple.contains("apple") { + build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed"); + } + spawn_and_wait(build_cmd); + + for entry in fs::read_dir(build_dir.join("deps")).unwrap() { + let entry = entry.unwrap(); + if let Some(ext) = entry.path().extension() { + if ext == "rmeta" || ext == "d" || ext == "dSYM" || ext == "clif" { + continue; + } + } else { + continue; + }; + target_libs.libs.push(entry.path()); + } + + target_libs +} + +fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> { + if !config::get_bool("keep_sysroot") { + crate::prepare::prepare_stdlib(dirs, &compiler.rustc); + } + + if !compiler.triple.ends_with("windows-gnu") { + return None; + } + + RTSTARTUP_SYSROOT.ensure_fresh(dirs); + + let rtstartup_src = STDLIB_SRC.to_path(dirs).join("library").join("rtstartup"); + let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; + + for file in ["rsbegin", "rsend"] { + let obj = RTSTARTUP_SYSROOT.to_path(dirs).join(format!("{file}.o")); + let mut build_rtstartup_cmd = Command::new(&compiler.rustc); + build_rtstartup_cmd + .arg("--target") + .arg(&compiler.triple) + .arg("--emit=obj") + .arg("-o") + .arg(&obj) + .arg(rtstartup_src.join(format!("{file}.rs"))); + spawn_and_wait(build_rtstartup_cmd); + target_libs.libs.push(obj.clone()); + } + + Some(target_libs) +} diff --git a/compiler/rustc_codegen_cranelift/build_system/config.rs b/compiler/rustc_codegen_cranelift/build_system/config.rs new file mode 100644 index 00000000000..ef540cf1f82 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/config.rs @@ -0,0 +1,55 @@ +use std::{fs, process}; + +fn load_config_file() -> Vec<(String, Option<String>)> { + fs::read_to_string("config.txt") + .unwrap() + .lines() + .map(|line| if let Some((line, _comment)) = line.split_once('#') { line } else { line }) + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + .map(|line| { + if let Some((key, val)) = line.split_once('=') { + (key.trim().to_owned(), Some(val.trim().to_owned())) + } else { + (line.to_owned(), None) + } + }) + .collect() +} + +pub(crate) fn get_bool(name: &str) -> bool { + let values = load_config_file() + .into_iter() + .filter(|(key, _)| key == name) + .map(|(_, val)| val) + .collect::<Vec<_>>(); + if values.is_empty() { + false + } else { + if values.iter().any(|val| val.is_some()) { + eprintln!("Boolean config `{}` has a value", name); + process::exit(1); + } + true + } +} + +pub(crate) fn get_value(name: &str) -> Option<String> { + let values = load_config_file() + .into_iter() + .filter(|(key, _)| key == name) + .map(|(_, val)| val) + .collect::<Vec<_>>(); + if values.is_empty() { + None + } else if values.len() == 1 { + if values[0].is_none() { + eprintln!("Config `{}` missing value", name); + process::exit(1); + } + values.into_iter().next().unwrap() + } else { + eprintln!("Config `{}` given multiple values: {:?}", name, values); + process::exit(1); + } +} diff --git a/compiler/rustc_codegen_cranelift/build_system/main.rs b/compiler/rustc_codegen_cranelift/build_system/main.rs new file mode 100644 index 00000000000..9ddeda583af --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/main.rs @@ -0,0 +1,269 @@ +#![warn(rust_2018_idioms)] +#![warn(unused_lifetimes)] +#![warn(unreachable_pub)] + +use std::path::PathBuf; +use std::{env, process}; + +use self::utils::Compiler; + +mod abi_cafe; +mod bench; +mod build_backend; +mod build_sysroot; +mod config; +mod path; +mod prepare; +mod rustc_info; +mod shared_utils; +mod tests; +mod utils; + +fn usage() { + eprintln!("{}", include_str!("usage.txt")); +} + +macro_rules! arg_error { + ($($err:tt)*) => {{ + eprintln!($($err)*); + usage(); + std::process::exit(1); + }}; +} + +#[derive(PartialEq, Debug)] +enum Command { + Prepare, + Build, + Test, + AbiCafe, + Bench, +} + +#[derive(Copy, Clone, Debug)] +enum SysrootKind { + None, + Clif, + Llvm, +} + +#[derive(Clone, Debug)] +enum CodegenBackend { + Local(PathBuf), + Builtin(String), +} + +fn main() { + if env::var_os("RUST_BACKTRACE").is_none() { + env::set_var("RUST_BACKTRACE", "1"); + } + env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1"); + + // Force incr comp even in release mode unless in CI or incremental builds are explicitly disabled + if env::var_os("CARGO_BUILD_INCREMENTAL").is_none() { + env::set_var("CARGO_BUILD_INCREMENTAL", "true"); + } + + let mut args = env::args().skip(1); + let command = match args.next().as_deref() { + Some("prepare") => Command::Prepare, + Some("build") => Command::Build, + Some("test") => Command::Test, + Some("abi-cafe") => Command::AbiCafe, + Some("bench") => Command::Bench, + Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag), + Some(command) => arg_error!("Unknown command {}", command), + None => { + usage(); + process::exit(0); + } + }; + + let mut out_dir = PathBuf::from("."); + let mut download_dir = None; + let mut channel = "release"; + let mut sysroot_kind = SysrootKind::Clif; + let mut use_unstable_features = true; + let mut frozen = false; + let mut skip_tests = vec![]; + let mut use_backend = None; + while let Some(arg) = args.next().as_deref() { + match arg { + "--out-dir" => { + out_dir = PathBuf::from(args.next().unwrap_or_else(|| { + arg_error!("--out-dir requires argument"); + })); + } + "--download-dir" => { + download_dir = Some(PathBuf::from(args.next().unwrap_or_else(|| { + arg_error!("--download-dir requires argument"); + }))); + } + "--debug" => channel = "debug", + "--sysroot" => { + sysroot_kind = match args.next().as_deref() { + Some("none") => SysrootKind::None, + Some("clif") => SysrootKind::Clif, + Some("llvm") => SysrootKind::Llvm, + Some(arg) => arg_error!("Unknown sysroot kind {}", arg), + None => arg_error!("--sysroot requires argument"), + } + } + "--no-unstable-features" => use_unstable_features = false, + "--frozen" => frozen = true, + "--skip-test" => { + // FIXME check that all passed in tests actually exist + skip_tests.push(args.next().unwrap_or_else(|| { + arg_error!("--skip-test requires argument"); + })); + } + "--use-backend" => { + use_backend = Some(match args.next() { + Some(name) => name, + None => arg_error!("--use-backend requires argument"), + }); + } + flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag), + arg => arg_error!("Unexpected argument {}", arg), + } + } + + let current_dir = std::env::current_dir().unwrap(); + out_dir = current_dir.join(out_dir); + + if command == Command::Prepare { + prepare::prepare(&path::Dirs { + source_dir: current_dir.clone(), + download_dir: download_dir + .map(|dir| current_dir.join(dir)) + .unwrap_or_else(|| out_dir.join("download")), + build_dir: PathBuf::from("dummy_do_not_use"), + dist_dir: PathBuf::from("dummy_do_not_use"), + frozen, + }); + process::exit(0); + } + + let rustup_toolchain_name = match (env::var("CARGO"), env::var("RUSTC"), env::var("RUSTDOC")) { + (Ok(_), Ok(_), Ok(_)) => None, + (_, Err(_), Err(_)) => Some(rustc_info::get_toolchain_name()), + vars => { + eprintln!( + "If RUSTC or RUSTDOC is set, both need to be set and in addition CARGO needs to be set: {vars:?}" + ); + process::exit(1); + } + }; + let bootstrap_host_compiler = { + let cargo = rustc_info::get_cargo_path(); + let rustc = rustc_info::get_rustc_path(); + let rustdoc = rustc_info::get_rustdoc_path(); + let triple = std::env::var("HOST_TRIPLE") + .ok() + .or_else(|| config::get_value("host")) + .unwrap_or_else(|| rustc_info::get_host_triple(&rustc)); + Compiler { + cargo, + rustc, + rustdoc, + rustflags: vec![], + rustdocflags: vec![], + triple, + runner: vec![], + } + }; + let target_triple = std::env::var("TARGET_TRIPLE") + .ok() + .or_else(|| config::get_value("target")) + .unwrap_or_else(|| bootstrap_host_compiler.triple.clone()); + + let dirs = path::Dirs { + source_dir: current_dir.clone(), + download_dir: download_dir + .map(|dir| current_dir.join(dir)) + .unwrap_or_else(|| out_dir.join("download")), + build_dir: out_dir.join("build"), + dist_dir: out_dir.join("dist"), + frozen, + }; + + path::RelPath::BUILD.ensure_exists(&dirs); + + { + // Make sure we always explicitly specify the target dir + let target = + path::RelPath::BUILD.join("target_dir_should_be_set_explicitly").to_path(&dirs); + env::set_var("CARGO_TARGET_DIR", &target); + let _ = std::fs::remove_file(&target); + std::fs::File::create(target).unwrap(); + } + + env::set_var("RUSTC", "rustc_should_be_set_explicitly"); + env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly"); + + let cg_clif_dylib = if let Some(name) = use_backend { + CodegenBackend::Builtin(name) + } else { + CodegenBackend::Local(build_backend::build_backend( + &dirs, + channel, + &bootstrap_host_compiler, + use_unstable_features, + )) + }; + match command { + Command::Prepare => { + // Handled above + } + Command::Test => { + tests::run_tests( + &dirs, + channel, + sysroot_kind, + use_unstable_features, + &skip_tests.iter().map(|test| &**test).collect::<Vec<_>>(), + &cg_clif_dylib, + &bootstrap_host_compiler, + rustup_toolchain_name.as_deref(), + target_triple.clone(), + ); + } + Command::AbiCafe => { + if bootstrap_host_compiler.triple != target_triple { + eprintln!("Abi-cafe doesn't support cross-compilation"); + process::exit(1); + } + abi_cafe::run( + channel, + sysroot_kind, + &dirs, + &cg_clif_dylib, + rustup_toolchain_name.as_deref(), + &bootstrap_host_compiler, + ); + } + Command::Build => { + build_sysroot::build_sysroot( + &dirs, + channel, + sysroot_kind, + &cg_clif_dylib, + &bootstrap_host_compiler, + rustup_toolchain_name.as_deref(), + target_triple, + ); + } + Command::Bench => { + build_sysroot::build_sysroot( + &dirs, + channel, + sysroot_kind, + &cg_clif_dylib, + &bootstrap_host_compiler, + rustup_toolchain_name.as_deref(), + target_triple, + ); + bench::benchmark(&dirs, &bootstrap_host_compiler); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/build_system/path.rs b/compiler/rustc_codegen_cranelift/build_system/path.rs new file mode 100644 index 00000000000..8572815fc55 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/path.rs @@ -0,0 +1,70 @@ +use std::fs; +use std::path::PathBuf; + +use crate::utils::remove_dir_if_exists; + +#[derive(Debug, Clone)] +pub(crate) struct Dirs { + pub(crate) source_dir: PathBuf, + pub(crate) download_dir: PathBuf, + pub(crate) build_dir: PathBuf, + pub(crate) dist_dir: PathBuf, + pub(crate) frozen: bool, +} + +#[doc(hidden)] +#[derive(Debug, Copy, Clone)] +pub(crate) enum PathBase { + Source, + Download, + Build, + Dist, +} + +impl PathBase { + fn to_path(self, dirs: &Dirs) -> PathBuf { + match self { + PathBase::Source => dirs.source_dir.clone(), + PathBase::Download => dirs.download_dir.clone(), + PathBase::Build => dirs.build_dir.clone(), + PathBase::Dist => dirs.dist_dir.clone(), + } + } +} + +#[derive(Debug, Copy, Clone)] +pub(crate) enum RelPath { + Base(PathBase), + Join(&'static RelPath, &'static str), +} + +impl RelPath { + pub(crate) const SOURCE: RelPath = RelPath::Base(PathBase::Source); + pub(crate) const DOWNLOAD: RelPath = RelPath::Base(PathBase::Download); + pub(crate) const BUILD: RelPath = RelPath::Base(PathBase::Build); + pub(crate) const DIST: RelPath = RelPath::Base(PathBase::Dist); + + pub(crate) const SCRIPTS: RelPath = RelPath::SOURCE.join("scripts"); + pub(crate) const PATCHES: RelPath = RelPath::SOURCE.join("patches"); + + pub(crate) const fn join(&'static self, suffix: &'static str) -> RelPath { + RelPath::Join(self, suffix) + } + + pub(crate) fn to_path(&self, dirs: &Dirs) -> PathBuf { + match self { + RelPath::Base(base) => base.to_path(dirs), + RelPath::Join(base, suffix) => base.to_path(dirs).join(suffix), + } + } + + pub(crate) fn ensure_exists(&self, dirs: &Dirs) { + fs::create_dir_all(self.to_path(dirs)).unwrap(); + } + + pub(crate) fn ensure_fresh(&self, dirs: &Dirs) { + let path = self.to_path(dirs); + remove_dir_if_exists(&path); + fs::create_dir_all(path).unwrap(); + } +} diff --git a/compiler/rustc_codegen_cranelift/build_system/prepare.rs b/compiler/rustc_codegen_cranelift/build_system/prepare.rs new file mode 100644 index 00000000000..be0bed0f4e6 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/prepare.rs @@ -0,0 +1,287 @@ +use std::ffi::OsStr; +use std::fs; +use std::hash::{Hash, Hasher}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::build_sysroot::STDLIB_SRC; +use crate::path::{Dirs, RelPath}; +use crate::rustc_info::get_default_sysroot; +use crate::utils::{ + copy_dir_recursively, git_command, remove_dir_if_exists, retry_spawn_and_wait, spawn_and_wait, +}; + +pub(crate) fn prepare(dirs: &Dirs) { + RelPath::DOWNLOAD.ensure_exists(dirs); + crate::tests::RAND_REPO.fetch(dirs); + crate::tests::REGEX_REPO.fetch(dirs); +} + +pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { + let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust"); + assert!(sysroot_src_orig.exists()); + + apply_patches(dirs, "stdlib", &sysroot_src_orig, &STDLIB_SRC.to_path(dirs)); +} + +pub(crate) struct GitRepo { + url: GitRepoUrl, + rev: &'static str, + content_hash: &'static str, + patch_name: &'static str, +} + +enum GitRepoUrl { + Github { user: &'static str, repo: &'static str }, +} + +// Note: This uses a hasher which is not cryptographically secure. This is fine as the hash is meant +// to protect against accidental modification and outdated downloads, not against manipulation. +fn hash_file(file: &std::path::Path) -> u64 { + let contents = std::fs::read(file).unwrap(); + #[allow(deprecated)] + let mut hasher = std::hash::SipHasher::new(); + // The following is equivalent to + // std::hash::Hash::hash(&contents, &mut hasher); + // but gives the same result independent of host byte order. + hasher.write_usize(contents.len().to_le()); + Hash::hash_slice(&contents, &mut hasher); + std::hash::Hasher::finish(&hasher) +} + +fn hash_dir(dir: &std::path::Path) -> u64 { + let mut sub_hashes = std::collections::BTreeMap::new(); + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + sub_hashes.insert( + entry.file_name().to_str().unwrap().to_owned(), + hash_dir(&entry.path()).to_le(), + ); + } else { + sub_hashes.insert( + entry.file_name().to_str().unwrap().to_owned(), + hash_file(&entry.path()).to_le(), + ); + } + } + #[allow(deprecated)] + let mut hasher = std::hash::SipHasher::new(); + // The following is equivalent to + // std::hash::Hash::hash(&sub_hashes, &mut hasher); + // but gives the same result independent of host byte order. + hasher.write_usize(sub_hashes.len().to_le()); + for elt in sub_hashes { + elt.hash(&mut hasher); + } + std::hash::Hasher::finish(&hasher) +} + +impl GitRepo { + pub(crate) const fn github( + user: &'static str, + repo: &'static str, + rev: &'static str, + content_hash: &'static str, + patch_name: &'static str, + ) -> GitRepo { + GitRepo { url: GitRepoUrl::Github { user, repo }, rev, content_hash, patch_name } + } + + fn download_dir(&self, dirs: &Dirs) -> PathBuf { + match self.url { + GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs), + } + } + + pub(crate) const fn source_dir(&self) -> RelPath { + match self.url { + GitRepoUrl::Github { user: _, repo } => RelPath::BUILD.join(repo), + } + } + + pub(crate) fn fetch(&self, dirs: &Dirs) { + let download_dir = self.download_dir(dirs); + + if download_dir.exists() { + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); + if actual_hash == self.content_hash { + eprintln!("[FRESH] {}", download_dir.display()); + return; + } else { + eprintln!( + "Mismatched content hash for {download_dir}: {actual_hash} != {content_hash}. Downloading again.", + download_dir = download_dir.display(), + content_hash = self.content_hash, + ); + } + } + + match self.url { + GitRepoUrl::Github { user, repo } => { + clone_repo_shallow_github(dirs, &download_dir, user, repo, self.rev); + } + } + + let source_lockfile = + RelPath::PATCHES.to_path(dirs).join(format!("{}-lock.toml", self.patch_name)); + let target_lockfile = download_dir.join("Cargo.lock"); + if source_lockfile.exists() { + assert!(!target_lockfile.exists()); + fs::copy(source_lockfile, target_lockfile).unwrap(); + } else { + assert!(target_lockfile.exists()); + } + + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); + if actual_hash != self.content_hash { + eprintln!( + "Download of {download_dir} failed with mismatched content hash: {actual_hash} != {content_hash}", + download_dir = download_dir.display(), + content_hash = self.content_hash, + ); + std::process::exit(1); + } + } + + pub(crate) fn patch(&self, dirs: &Dirs) { + apply_patches( + dirs, + self.patch_name, + &self.download_dir(dirs), + &self.source_dir().to_path(dirs), + ); + } +} + +#[allow(dead_code)] +fn clone_repo(download_dir: &Path, repo: &str, rev: &str) { + eprintln!("[CLONE] {}", repo); + // Ignore exit code as the repo may already have been checked out + git_command(None, "clone").arg(repo).arg(download_dir).spawn().unwrap().wait().unwrap(); + + let mut clean_cmd = git_command(download_dir, "checkout"); + clean_cmd.arg("--").arg("."); + spawn_and_wait(clean_cmd); + + let mut checkout_cmd = git_command(download_dir, "checkout"); + checkout_cmd.arg("-q").arg(rev); + spawn_and_wait(checkout_cmd); + + std::fs::remove_dir_all(download_dir.join(".git")).unwrap(); +} + +fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: &str, rev: &str) { + if cfg!(windows) { + // Older windows doesn't have tar or curl by default. Fall back to using git. + clone_repo(download_dir, &format!("https://github.com/{}/{}.git", user, repo), rev); + return; + } + + let archive_url = format!("https://github.com/{}/{}/archive/{}.tar.gz", user, repo, rev); + let archive_file = RelPath::DOWNLOAD.to_path(dirs).join(format!("{}.tar.gz", rev)); + let archive_dir = RelPath::DOWNLOAD.to_path(dirs).join(format!("{}-{}", repo, rev)); + + eprintln!("[DOWNLOAD] {}/{} from {}", user, repo, archive_url); + + // Remove previous results if they exists + let _ = std::fs::remove_file(&archive_file); + let _ = std::fs::remove_dir_all(&archive_dir); + let _ = std::fs::remove_dir_all(&download_dir); + + // Download zip archive + let mut download_cmd = Command::new("curl"); + download_cmd + .arg("--max-time") + .arg("600") + .arg("-y") + .arg("30") + .arg("-Y") + .arg("10") + .arg("--connect-timeout") + .arg("30") + .arg("--continue-at") + .arg("-") + .arg("--location") + .arg("--output") + .arg(&archive_file) + .arg(archive_url); + retry_spawn_and_wait(5, download_cmd); + + // Unpack tar archive + let mut unpack_cmd = Command::new("tar"); + unpack_cmd.arg("xf").arg(&archive_file).current_dir(RelPath::DOWNLOAD.to_path(dirs)); + spawn_and_wait(unpack_cmd); + + // Rename unpacked dir to the expected name + std::fs::rename(archive_dir, &download_dir).unwrap(); + + // Cleanup + std::fs::remove_file(archive_file).unwrap(); +} + +fn init_git_repo(repo_dir: &Path) { + let mut git_init_cmd = git_command(repo_dir, "init"); + git_init_cmd.arg("-q"); + spawn_and_wait(git_init_cmd); + + let mut git_add_cmd = git_command(repo_dir, "add"); + git_add_cmd.arg("."); + spawn_and_wait(git_add_cmd); + + let mut git_commit_cmd = git_command(repo_dir, "commit"); + git_commit_cmd.arg("-m").arg("Initial commit").arg("-q"); + spawn_and_wait(git_commit_cmd); +} + +fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec<PathBuf> { + let mut patches: Vec<_> = fs::read_dir(RelPath::PATCHES.to_path(dirs)) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension() == Some(OsStr::new("patch"))) + .filter(|path| { + path.file_name() + .unwrap() + .to_str() + .unwrap() + .split_once("-") + .unwrap() + .1 + .starts_with(crate_name) + }) + .collect(); + patches.sort(); + patches +} + +pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) { + // FIXME avoid copy and patch if src, patches and target are unchanged + + eprintln!("[COPY] {crate_name} source"); + + remove_dir_if_exists(target_dir); + fs::create_dir_all(target_dir).unwrap(); + if crate_name == "stdlib" { + fs::create_dir(target_dir.join("library")).unwrap(); + copy_dir_recursively(&source_dir.join("library"), &target_dir.join("library")); + } else { + copy_dir_recursively(source_dir, target_dir); + } + + init_git_repo(target_dir); + + if crate_name == "<none>" { + return; + } + + for patch in get_patches(dirs, crate_name) { + eprintln!( + "[PATCH] {:?} <- {:?}", + target_dir.file_name().unwrap(), + patch.file_name().unwrap() + ); + let mut apply_patch_cmd = git_command(target_dir, "am"); + apply_patch_cmd.arg(patch).arg("-q"); + spawn_and_wait(apply_patch_cmd); + } +} diff --git a/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs b/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs new file mode 100644 index 00000000000..5b71504e90a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs @@ -0,0 +1,99 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +pub(crate) fn get_host_triple(rustc: &Path) -> String { + let version_info = + Command::new(rustc).stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout; + String::from_utf8(version_info) + .unwrap() + .lines() + .to_owned() + .find(|line| line.starts_with("host")) + .unwrap() + .split(":") + .nth(1) + .unwrap() + .trim() + .to_owned() +} + +pub(crate) fn get_toolchain_name() -> String { + let active_toolchain = Command::new("rustup") + .stderr(Stdio::inherit()) + .args(&["show", "active-toolchain"]) + .output() + .unwrap() + .stdout; + String::from_utf8(active_toolchain).unwrap().trim().split_once(' ').unwrap().0.to_owned() +} + +pub(crate) fn get_cargo_path() -> PathBuf { + if let Ok(cargo) = std::env::var("CARGO") { + return PathBuf::from(cargo); + } + let cargo_path = Command::new("rustup") + .stderr(Stdio::inherit()) + .args(&["which", "cargo"]) + .output() + .unwrap() + .stdout; + Path::new(String::from_utf8(cargo_path).unwrap().trim()).to_owned() +} + +pub(crate) fn get_rustc_path() -> PathBuf { + if let Ok(rustc) = std::env::var("RUSTC") { + return PathBuf::from(rustc); + } + let rustc_path = Command::new("rustup") + .stderr(Stdio::inherit()) + .args(&["which", "rustc"]) + .output() + .unwrap() + .stdout; + Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned() +} + +pub(crate) fn get_rustdoc_path() -> PathBuf { + if let Ok(rustdoc) = std::env::var("RUSTDOC") { + return PathBuf::from(rustdoc); + } + let rustc_path = Command::new("rustup") + .stderr(Stdio::inherit()) + .args(&["which", "rustdoc"]) + .output() + .unwrap() + .stdout; + Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned() +} + +pub(crate) fn get_default_sysroot(rustc: &Path) -> PathBuf { + let default_sysroot = Command::new(rustc) + .stderr(Stdio::inherit()) + .args(&["--print", "sysroot"]) + .output() + .unwrap() + .stdout; + Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned() +} + +// FIXME call once for each target and pass result around in struct +pub(crate) fn get_file_name(rustc: &Path, crate_name: &str, crate_type: &str) -> String { + let file_name = Command::new(rustc) + .stderr(Stdio::inherit()) + .args(&[ + "--crate-name", + crate_name, + "--crate-type", + crate_type, + "--print", + "file-names", + "-", + ]) + .output() + .unwrap() + .stdout; + let file_name = String::from_utf8(file_name).unwrap().trim().to_owned(); + assert!(!file_name.contains('\n')); + assert!(file_name.contains(crate_name)); + file_name +} diff --git a/compiler/rustc_codegen_cranelift/build_system/shared_utils.rs b/compiler/rustc_codegen_cranelift/build_system/shared_utils.rs new file mode 100644 index 00000000000..0aea545ff7d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/shared_utils.rs @@ -0,0 +1,26 @@ +// This file is used by both the build system as well as cargo-clif.rs + +// Adapted from https://github.com/rust-lang/cargo/blob/6dc1deaddf62c7748c9097c7ea88e9ec77ff1a1a/src/cargo/core/compiler/build_context/target_info.rs#L750-L77 +pub(crate) fn rustflags_from_env(kind: &str) -> Vec<String> { + // First try CARGO_ENCODED_RUSTFLAGS from the environment. + // Prefer this over RUSTFLAGS since it's less prone to encoding errors. + if let Ok(a) = std::env::var(format!("CARGO_ENCODED_{}", kind)) { + if a.is_empty() { + return Vec::new(); + } + return a.split('\x1f').map(str::to_string).collect(); + } + + // Then try RUSTFLAGS from the environment + if let Ok(a) = std::env::var(kind) { + let args = a.split(' ').map(str::trim).filter(|s| !s.is_empty()).map(str::to_string); + return args.collect(); + } + + // No rustflags to be collected from the environment + Vec::new() +} + +pub(crate) fn rustflags_to_cmd_env(cmd: &mut std::process::Command, kind: &str, flags: &[String]) { + cmd.env(format!("CARGO_ENCODED_{}", kind), flags.join("\x1f")); +} diff --git a/compiler/rustc_codegen_cranelift/build_system/tests.rs b/compiler/rustc_codegen_cranelift/build_system/tests.rs new file mode 100644 index 00000000000..38c3786a94a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/tests.rs @@ -0,0 +1,484 @@ +use std::ffi::OsStr; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::path::{Dirs, RelPath}; +use crate::prepare::{apply_patches, GitRepo}; +use crate::rustc_info::get_default_sysroot; +use crate::shared_utils::rustflags_from_env; +use crate::utils::{spawn_and_wait, CargoProject, Compiler, LogGroup}; +use crate::{build_sysroot, config, CodegenBackend, SysrootKind}; + +static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example"); + +struct TestCase { + config: &'static str, + cmd: TestCaseCmd, +} + +enum TestCaseCmd { + Custom { func: &'static dyn Fn(&TestRunner<'_>) }, + BuildLib { source: &'static str, crate_types: &'static str }, + BuildBin { source: &'static str }, + BuildBinAndRun { source: &'static str, args: &'static [&'static str] }, + JitBin { source: &'static str, args: &'static str }, +} + +impl TestCase { + // FIXME reduce usage of custom test case commands + const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner<'_>)) -> Self { + Self { config, cmd: TestCaseCmd::Custom { func } } + } + + const fn build_lib( + config: &'static str, + source: &'static str, + crate_types: &'static str, + ) -> Self { + Self { config, cmd: TestCaseCmd::BuildLib { source, crate_types } } + } + + const fn build_bin(config: &'static str, source: &'static str) -> Self { + Self { config, cmd: TestCaseCmd::BuildBin { source } } + } + + const fn build_bin_and_run( + config: &'static str, + source: &'static str, + args: &'static [&'static str], + ) -> Self { + Self { config, cmd: TestCaseCmd::BuildBinAndRun { source, args } } + } + + const fn jit_bin(config: &'static str, source: &'static str, args: &'static str) -> Self { + Self { config, cmd: TestCaseCmd::JitBin { source, args } } + } +} + +const NO_SYSROOT_SUITE: &[TestCase] = &[ + TestCase::build_lib("build.mini_core", "example/mini_core.rs", "lib,dylib"), + TestCase::build_lib("build.example", "example/example.rs", "lib"), + TestCase::jit_bin("jit.mini_core_hello_world", "example/mini_core_hello_world.rs", "abc bcd"), + TestCase::build_bin_and_run( + "aot.mini_core_hello_world", + "example/mini_core_hello_world.rs", + &["abc", "bcd"], + ), +]; + +const BASE_SYSROOT_SUITE: &[TestCase] = &[ + TestCase::build_bin_and_run( + "aot.arbitrary_self_types_pointers_and_wrappers", + "example/arbitrary_self_types_pointers_and_wrappers.rs", + &[], + ), + TestCase::build_lib("build.alloc_system", "example/alloc_system.rs", "lib"), + TestCase::build_bin_and_run("aot.alloc_example", "example/alloc_example.rs", &[]), + TestCase::jit_bin("jit.std_example", "example/std_example.rs", "arg"), + TestCase::build_bin_and_run("aot.std_example", "example/std_example.rs", &["arg"]), + TestCase::build_bin_and_run("aot.dst_field_align", "example/dst-field-align.rs", &[]), + TestCase::build_bin_and_run( + "aot.subslice-patterns-const-eval", + "example/subslice-patterns-const-eval.rs", + &[], + ), + TestCase::build_bin_and_run( + "aot.track-caller-attribute", + "example/track-caller-attribute.rs", + &[], + ), + TestCase::build_bin_and_run("aot.float-minmax-pass", "example/float-minmax-pass.rs", &[]), + TestCase::build_bin_and_run("aot.mod_bench", "example/mod_bench.rs", &[]), + TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]), + TestCase::build_bin("aot.issue-59326", "example/issue-59326.rs"), + TestCase::custom("aot.polymorphize_coroutine", &|runner| { + runner.run_rustc(&["example/polymorphize_coroutine.rs", "-Zpolymorphize"]); + runner.run_out_command("polymorphize_coroutine", &[]); + }), + TestCase::build_bin_and_run("aot.neon", "example/neon.rs", &[]), + TestCase::custom("aot.gen_block_iterate", &|runner| { + runner.run_rustc([ + "example/gen_block_iterate.rs", + "--edition", + "2024", + "-Zunstable-options", + ]); + runner.run_out_command("gen_block_iterate", &[]); + }), + TestCase::build_bin_and_run("aot.raw-dylib", "example/raw-dylib.rs", &[]), +]; + +pub(crate) static RAND_REPO: GitRepo = GitRepo::github( + "rust-random", + "rand", + "1f4507a8e1cf8050e4ceef95eeda8f64645b6719", + "981f8bf489338978", + "rand", +); + +pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand_target"); + +pub(crate) static REGEX_REPO: GitRepo = GitRepo::github( + "rust-lang", + "regex", + "061ee815ef2c44101dba7b0b124600fcb03c1912", + "dc26aefbeeac03ca", + "regex", +); + +pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex_target"); + +pub(crate) static PORTABLE_SIMD_SRC: RelPath = RelPath::BUILD.join("coretests"); + +pub(crate) static PORTABLE_SIMD: CargoProject = + CargoProject::new(&PORTABLE_SIMD_SRC, "portable-simd_target"); + +static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests"); + +static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "coretests_target"); + +const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ + TestCase::custom("test.rust-random/rand", &|runner| { + RAND_REPO.patch(&runner.dirs); + + RAND.clean(&runner.dirs); + + if runner.is_native { + let mut test_cmd = RAND.test(&runner.target_compiler, &runner.dirs); + test_cmd.arg("--workspace").arg("--").arg("-q"); + spawn_and_wait(test_cmd); + } else { + eprintln!("Cross-Compiling: Not running tests"); + let mut build_cmd = RAND.build(&runner.target_compiler, &runner.dirs); + build_cmd.arg("--workspace").arg("--tests"); + spawn_and_wait(build_cmd); + } + }), + TestCase::custom("test.libcore", &|runner| { + apply_patches( + &runner.dirs, + "coretests", + &runner.stdlib_source.join("library/core/tests"), + &LIBCORE_TESTS_SRC.to_path(&runner.dirs), + ); + + let source_lockfile = RelPath::PATCHES.to_path(&runner.dirs).join("coretests-lock.toml"); + let target_lockfile = LIBCORE_TESTS_SRC.to_path(&runner.dirs).join("Cargo.lock"); + fs::copy(source_lockfile, target_lockfile).unwrap(); + + LIBCORE_TESTS.clean(&runner.dirs); + + if runner.is_native { + let mut test_cmd = LIBCORE_TESTS.test(&runner.target_compiler, &runner.dirs); + test_cmd.arg("--").arg("-q"); + spawn_and_wait(test_cmd); + } else { + eprintln!("Cross-Compiling: Not running tests"); + let mut build_cmd = LIBCORE_TESTS.build(&runner.target_compiler, &runner.dirs); + build_cmd.arg("--tests"); + spawn_and_wait(build_cmd); + } + }), + TestCase::custom("test.regex", &|runner| { + REGEX_REPO.patch(&runner.dirs); + + REGEX.clean(&runner.dirs); + + if runner.is_native { + let mut run_cmd = REGEX.test(&runner.target_compiler, &runner.dirs); + // regex-capi and regex-debug don't have any tests. Nor do they contain any code + // that is useful to test with cg_clif. Skip building them to reduce test time. + run_cmd.args([ + "-p", + "regex", + "-p", + "regex-syntax", + "--release", + "--all-targets", + "--", + "-q", + ]); + spawn_and_wait(run_cmd); + + let mut run_cmd = REGEX.test(&runner.target_compiler, &runner.dirs); + // don't run integration tests for regex-autonata. they take like 2min each without + // much extra coverage of simd usage. + run_cmd.args(["-p", "regex-automata", "--release", "--lib", "--", "-q"]); + spawn_and_wait(run_cmd); + } else { + eprintln!("Cross-Compiling: Not running tests"); + let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs); + build_cmd.arg("--tests"); + spawn_and_wait(build_cmd); + } + }), + TestCase::custom("test.portable-simd", &|runner| { + apply_patches( + &runner.dirs, + "portable-simd", + &runner.stdlib_source.join("library/portable-simd"), + &PORTABLE_SIMD_SRC.to_path(&runner.dirs), + ); + + PORTABLE_SIMD.clean(&runner.dirs); + + let mut build_cmd = PORTABLE_SIMD.build(&runner.target_compiler, &runner.dirs); + build_cmd.arg("--all-targets"); + spawn_and_wait(build_cmd); + + if runner.is_native { + let mut test_cmd = PORTABLE_SIMD.test(&runner.target_compiler, &runner.dirs); + test_cmd.arg("-q"); + // FIXME remove after portable-simd update + test_cmd + .arg("--") + .arg("--skip") + .arg("core_simd::swizzle::simd_swizzle") + .arg("--skip") + .arg("core_simd::vector::Simd<T,N>::lanes"); + spawn_and_wait(test_cmd); + } + }), +]; + +pub(crate) fn run_tests( + dirs: &Dirs, + channel: &str, + sysroot_kind: SysrootKind, + use_unstable_features: bool, + skip_tests: &[&str], + cg_clif_dylib: &CodegenBackend, + bootstrap_host_compiler: &Compiler, + rustup_toolchain_name: Option<&str>, + target_triple: String, +) { + let stdlib_source = + get_default_sysroot(&bootstrap_host_compiler.rustc).join("lib/rustlib/src/rust"); + assert!(stdlib_source.exists()); + + if config::get_bool("testsuite.no_sysroot") && !skip_tests.contains(&"testsuite.no_sysroot") { + let target_compiler = build_sysroot::build_sysroot( + dirs, + channel, + SysrootKind::None, + cg_clif_dylib, + bootstrap_host_compiler, + rustup_toolchain_name, + target_triple.clone(), + ); + + let runner = TestRunner::new( + dirs.clone(), + target_compiler, + use_unstable_features, + skip_tests, + bootstrap_host_compiler.triple == target_triple, + stdlib_source.clone(), + ); + + BUILD_EXAMPLE_OUT_DIR.ensure_fresh(dirs); + runner.run_testsuite(NO_SYSROOT_SUITE); + } else { + eprintln!("[SKIP] no_sysroot tests"); + } + + let run_base_sysroot = config::get_bool("testsuite.base_sysroot") + && !skip_tests.contains(&"testsuite.base_sysroot"); + let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot") + && !skip_tests.contains(&"testsuite.extended_sysroot"); + + if run_base_sysroot || run_extended_sysroot { + let target_compiler = build_sysroot::build_sysroot( + dirs, + channel, + sysroot_kind, + cg_clif_dylib, + bootstrap_host_compiler, + rustup_toolchain_name, + target_triple.clone(), + ); + + let mut runner = TestRunner::new( + dirs.clone(), + target_compiler, + use_unstable_features, + skip_tests, + bootstrap_host_compiler.triple == target_triple, + stdlib_source, + ); + + if run_base_sysroot { + runner.run_testsuite(BASE_SYSROOT_SUITE); + } else { + eprintln!("[SKIP] base_sysroot tests"); + } + + if run_extended_sysroot { + // Rust's build system denies a couple of lints that trigger on several of the test + // projects. Changing the code to fix them is not worth it, so just silence all lints. + runner.target_compiler.rustflags.push("--cap-lints=allow".to_owned()); + runner.run_testsuite(EXTENDED_SYSROOT_SUITE); + } else { + eprintln!("[SKIP] extended_sysroot tests"); + } + } +} + +struct TestRunner<'a> { + is_native: bool, + jit_supported: bool, + skip_tests: &'a [&'a str], + dirs: Dirs, + target_compiler: Compiler, + stdlib_source: PathBuf, +} + +impl<'a> TestRunner<'a> { + fn new( + dirs: Dirs, + mut target_compiler: Compiler, + use_unstable_features: bool, + skip_tests: &'a [&'a str], + is_native: bool, + stdlib_source: PathBuf, + ) -> Self { + target_compiler.rustflags.extend(rustflags_from_env("RUSTFLAGS")); + target_compiler.rustdocflags.extend(rustflags_from_env("RUSTDOCFLAGS")); + + // FIXME fix `#[linkage = "extern_weak"]` without this + if target_compiler.triple.contains("darwin") { + target_compiler.rustflags.extend([ + "-Clink-arg=-undefined".to_owned(), + "-Clink-arg=dynamic_lookup".to_owned(), + ]); + } + + let jit_supported = use_unstable_features + && is_native + && target_compiler.triple.contains("x86_64") + && !target_compiler.triple.contains("windows"); + + Self { is_native, jit_supported, skip_tests, dirs, target_compiler, stdlib_source } + } + + fn run_testsuite(&self, tests: &[TestCase]) { + for TestCase { config, cmd } in tests { + let (tag, testname) = config.split_once('.').unwrap(); + let tag = tag.to_uppercase(); + let is_jit_test = tag == "JIT"; + + let _guard = if !config::get_bool(config) + || (is_jit_test && !self.jit_supported) + || self.skip_tests.contains(&config) + { + eprintln!("[{tag}] {testname} (skipped)"); + continue; + } else { + let guard = LogGroup::guard(&format!("[{tag}] {testname}")); + eprintln!("[{tag}] {testname}"); + guard + }; + + match *cmd { + TestCaseCmd::Custom { func } => func(self), + TestCaseCmd::BuildLib { source, crate_types } => { + self.run_rustc([source, "--crate-type", crate_types]); + } + TestCaseCmd::BuildBin { source } => { + self.run_rustc([source]); + } + TestCaseCmd::BuildBinAndRun { source, args } => { + self.run_rustc([source]); + self.run_out_command( + source.split('/').last().unwrap().split('.').next().unwrap(), + args, + ); + } + TestCaseCmd::JitBin { source, args } => { + let mut jit_cmd = self.rustc_command([ + "-Zunstable-options", + "-Cllvm-args=mode=jit", + "-Cprefer-dynamic", + source, + "--cfg", + "jit", + ]); + if !args.is_empty() { + jit_cmd.env("CG_CLIF_JIT_ARGS", args); + } + spawn_and_wait(jit_cmd); + + eprintln!("[JIT-lazy] {testname}"); + let mut jit_cmd = self.rustc_command([ + "-Zunstable-options", + "-Cllvm-args=mode=jit-lazy", + "-Cprefer-dynamic", + source, + "--cfg", + "jit", + ]); + if !args.is_empty() { + jit_cmd.env("CG_CLIF_JIT_ARGS", args); + } + spawn_and_wait(jit_cmd); + } + } + } + } + + #[must_use] + fn rustc_command<I, S>(&self, args: I) -> Command + where + I: IntoIterator<Item = S>, + S: AsRef<OsStr>, + { + let mut cmd = Command::new(&self.target_compiler.rustc); + cmd.args(&self.target_compiler.rustflags); + cmd.arg("-L"); + cmd.arg(format!("crate={}", BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).display())); + cmd.arg("--out-dir"); + cmd.arg(BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs)); + cmd.arg("-Cdebuginfo=2"); + cmd.arg("--target"); + cmd.arg(&self.target_compiler.triple); + cmd.arg("-Cpanic=abort"); + cmd.arg("-Zunstable-options"); + cmd.arg("--check-cfg=cfg(jit)"); + cmd.args(args); + cmd + } + + fn run_rustc<I, S>(&self, args: I) + where + I: IntoIterator<Item = S>, + S: AsRef<OsStr>, + { + spawn_and_wait(self.rustc_command(args)); + } + + fn run_out_command(&self, name: &str, args: &[&str]) { + let mut full_cmd = vec![]; + + // Prepend the RUN_WRAPPER's + if !self.target_compiler.runner.is_empty() { + full_cmd.extend(self.target_compiler.runner.iter().cloned()); + } + + full_cmd.push( + BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).join(name).to_str().unwrap().to_string(), + ); + + for arg in args { + full_cmd.push(arg.to_string()); + } + + let mut cmd_iter = full_cmd.into_iter(); + let first = cmd_iter.next().unwrap(); + + let mut cmd = Command::new(first); + cmd.args(cmd_iter); + + spawn_and_wait(cmd); + } +} diff --git a/compiler/rustc_codegen_cranelift/build_system/usage.txt b/compiler/rustc_codegen_cranelift/build_system/usage.txt new file mode 100644 index 00000000000..f652599447b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/usage.txt @@ -0,0 +1,49 @@ +The build system of cg_clif. + +USAGE: + ./y.sh prepare [--out-dir DIR] [--download-dir DIR] + ./y.sh build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] [--skip-test TESTNAME] + ./y.sh abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + +OPTIONS: + --debug + Build cg_clif and the standard library in debug mode rather than release mode. + Warning: An unoptimized cg_clif is very slow. + + --sysroot none|clif|llvm + Which sysroot libraries to use: + `none` will not include any standard library in the sysroot. + `clif` will build the standard library using Cranelift. + `llvm` will use the pre-compiled standard library of rustc which is compiled with LLVM. + + --out-dir DIR + Specify the directory in which the download, build and dist directories are stored. + By default this is the working directory. + + --download-dir DIR + Specify the directory in which the download directory is stored. Overrides --out-dir. + + --no-unstable-features + Some features are not yet ready for production usage. This option will disable these + features. This includes the JIT mode and inline assembly support. + + --frozen + Require Cargo.lock and cache are up to date + + --skip-test TESTNAME + Skip testing the TESTNAME test. The test name format is the same as config.txt. + + --use-backend NAME + Use the existing Cranelift (or other) backend of the rustc with which we built. + Warning: This is meant for use in rust's CI only! + +REQUIREMENTS: + * Rustup: By default rustup is used to install the right nightly version. If you don't want to + use rustup, you can manually install the nightly version indicated by rust-toolchain.toml and + point the CARGO, RUSTC and RUSTDOC env vars to the right executables. + * Git: Git is used for applying patches and on Windows for downloading test repos. + * Curl and tar (non-Windows only): Used by `./y.sh prepare` to download a single commit for + repos. Git will be used to clone the whole repo when using Windows. + * [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.sh bench`. diff --git a/compiler/rustc_codegen_cranelift/build_system/utils.rs b/compiler/rustc_codegen_cranelift/build_system/utils.rs new file mode 100644 index 00000000000..3c4b45e02cc --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_system/utils.rs @@ -0,0 +1,280 @@ +use std::path::{Path, PathBuf}; +use std::process::{self, Command}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::{env, fs, io}; + +use crate::path::{Dirs, RelPath}; +use crate::shared_utils::rustflags_to_cmd_env; + +#[derive(Clone, Debug)] +pub(crate) struct Compiler { + pub(crate) cargo: PathBuf, + pub(crate) rustc: PathBuf, + pub(crate) rustdoc: PathBuf, + pub(crate) rustflags: Vec<String>, + pub(crate) rustdocflags: Vec<String>, + pub(crate) triple: String, + pub(crate) runner: Vec<String>, +} + +impl Compiler { + pub(crate) fn set_cross_linker_and_runner(&mut self) { + match self.triple.as_str() { + "aarch64-unknown-linux-gnu" => { + // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. + self.rustflags.push("-Clinker=aarch64-linux-gnu-gcc".to_owned()); + self.rustdocflags.push("-Clinker=aarch64-linux-gnu-gcc".to_owned()); + self.runner = vec![ + "qemu-aarch64".to_owned(), + "-L".to_owned(), + "/usr/aarch64-linux-gnu".to_owned(), + ]; + } + "s390x-unknown-linux-gnu" => { + // We are cross-compiling for s390x. Use the correct linker and run tests in qemu. + self.rustflags.push("-Clinker=s390x-linux-gnu-gcc".to_owned()); + self.rustdocflags.push("-Clinker=s390x-linux-gnu-gcc".to_owned()); + self.runner = vec![ + "qemu-s390x".to_owned(), + "-L".to_owned(), + "/usr/s390x-linux-gnu".to_owned(), + ]; + } + "riscv64gc-unknown-linux-gnu" => { + // We are cross-compiling for riscv64. Use the correct linker and run tests in qemu. + self.rustflags.push("-Clinker=riscv64-linux-gnu-gcc".to_owned()); + self.rustdocflags.push("-Clinker=riscv64-linux-gnu-gcc".to_owned()); + self.runner = vec![ + "qemu-riscv64".to_owned(), + "-L".to_owned(), + "/usr/riscv64-linux-gnu".to_owned(), + ]; + } + "x86_64-pc-windows-gnu" => { + // We are cross-compiling for Windows. Run tests in wine. + self.runner = vec!["wine".to_owned()]; + } + _ => { + eprintln!("Unknown non-native platform"); + } + } + } +} + +pub(crate) struct CargoProject { + source: &'static RelPath, + target: &'static str, +} + +impl CargoProject { + pub(crate) const fn new(path: &'static RelPath, target: &'static str) -> CargoProject { + CargoProject { source: path, target } + } + + pub(crate) fn source_dir(&self, dirs: &Dirs) -> PathBuf { + self.source.to_path(dirs) + } + + pub(crate) fn manifest_path(&self, dirs: &Dirs) -> PathBuf { + self.source_dir(dirs).join("Cargo.toml") + } + + pub(crate) fn target_dir(&self, dirs: &Dirs) -> PathBuf { + RelPath::BUILD.join(self.target).to_path(dirs) + } + + #[must_use] + fn base_cmd(&self, command: &str, cargo: &Path, dirs: &Dirs) -> Command { + let mut cmd = Command::new(cargo); + + cmd.arg(command) + .arg("--manifest-path") + .arg(self.manifest_path(dirs)) + .arg("--target-dir") + .arg(self.target_dir(dirs)) + .arg("--locked"); + + if dirs.frozen { + cmd.arg("--frozen"); + } + + cmd + } + + #[must_use] + fn build_cmd(&self, command: &str, compiler: &Compiler, dirs: &Dirs) -> Command { + let mut cmd = self.base_cmd(command, &compiler.cargo, dirs); + + cmd.arg("--target").arg(&compiler.triple); + + cmd.env("RUSTC", &compiler.rustc); + cmd.env("RUSTDOC", &compiler.rustdoc); + rustflags_to_cmd_env(&mut cmd, "RUSTFLAGS", &compiler.rustflags); + rustflags_to_cmd_env(&mut cmd, "RUSTDOCFLAGS", &compiler.rustdocflags); + if !compiler.runner.is_empty() { + cmd.env( + format!("CARGO_TARGET_{}_RUNNER", compiler.triple.to_uppercase().replace('-', "_")), + compiler.runner.join(" "), + ); + } + + cmd + } + + pub(crate) fn clean(&self, dirs: &Dirs) { + let _ = fs::remove_dir_all(self.target_dir(dirs)); + } + + #[must_use] + pub(crate) fn build(&self, compiler: &Compiler, dirs: &Dirs) -> Command { + self.build_cmd("build", compiler, dirs) + } + + #[must_use] + pub(crate) fn test(&self, compiler: &Compiler, dirs: &Dirs) -> Command { + self.build_cmd("test", compiler, dirs) + } + + #[must_use] + pub(crate) fn run(&self, compiler: &Compiler, dirs: &Dirs) -> Command { + self.build_cmd("run", compiler, dirs) + } +} + +#[must_use] +pub(crate) fn hyperfine_command( + warmup: u64, + runs: u64, + prepare: Option<&str>, + cmds: &[(&str, &str)], + markdown_export: &Path, +) -> Command { + let mut bench = Command::new("hyperfine"); + + bench.arg("--export-markdown").arg(markdown_export); + + if warmup != 0 { + bench.arg("--warmup").arg(warmup.to_string()); + } + + if runs != 0 { + bench.arg("--runs").arg(runs.to_string()); + } + + if let Some(prepare) = prepare { + bench.arg("--prepare").arg(prepare); + } + + for &(name, cmd) in cmds { + if name != "" { + bench.arg("-n").arg(name); + } + bench.arg(cmd); + } + + bench +} + +#[must_use] +pub(crate) fn git_command<'a>(repo_dir: impl Into<Option<&'a Path>>, cmd: &str) -> Command { + let mut git_cmd = Command::new("git"); + git_cmd + .arg("-c") + .arg("user.name=Dummy") + .arg("-c") + .arg("user.email=dummy@example.com") + .arg("-c") + .arg("core.autocrlf=false") + .arg("-c") + .arg("commit.gpgSign=false") + .arg(cmd); + if let Some(repo_dir) = repo_dir.into() { + git_cmd.current_dir(repo_dir); + } + git_cmd +} + +#[track_caller] +pub(crate) fn try_hard_link(src: impl AsRef<Path>, dst: impl AsRef<Path>) { + let src = src.as_ref(); + let dst = dst.as_ref(); + if let Err(_) = fs::hard_link(src, dst) { + fs::copy(src, dst).unwrap(); // Fallback to copying if hardlinking failed + } +} + +#[track_caller] +pub(crate) fn spawn_and_wait(mut cmd: Command) { + let status = cmd.spawn().unwrap().wait().unwrap(); + if !status.success() { + eprintln!("{cmd:?} exited with status {:?}", status); + process::exit(1); + } +} + +// Based on the retry function in rust's src/ci/shared.sh +#[track_caller] +pub(crate) fn retry_spawn_and_wait(tries: u64, mut cmd: Command) { + for i in 1..tries + 1 { + if i != 1 { + eprintln!("Command failed. Attempt {i}/{tries}:"); + } + if cmd.spawn().unwrap().wait().unwrap().success() { + return; + } + std::thread::sleep(std::time::Duration::from_secs(i * 5)); + } + eprintln!("The command has failed after {tries} attempts."); + process::exit(1); +} + +pub(crate) fn remove_dir_if_exists(path: &Path) { + match fs::remove_dir_all(&path) { + Ok(()) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => panic!("Failed to remove {path}: {err}", path = path.display()), + } +} + +pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) { + for entry in fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let filename = entry.file_name(); + if filename == "." || filename == ".." { + continue; + } + if entry.metadata().unwrap().is_dir() { + fs::create_dir(to.join(&filename)).unwrap(); + copy_dir_recursively(&from.join(&filename), &to.join(&filename)); + } else { + fs::copy(from.join(&filename), to.join(&filename)).unwrap(); + } + } +} + +static IN_GROUP: AtomicBool = AtomicBool::new(false); +pub(crate) struct LogGroup { + is_gha: bool, +} + +impl LogGroup { + pub(crate) fn guard(name: &str) -> LogGroup { + let is_gha = env::var("GITHUB_ACTIONS").is_ok(); + + assert!(!IN_GROUP.swap(true, Ordering::SeqCst)); + if is_gha { + eprintln!("::group::{name}"); + } + + LogGroup { is_gha } + } +} + +impl Drop for LogGroup { + fn drop(&mut self) { + if self.is_gha { + eprintln!("::endgroup::"); + } + IN_GROUP.store(false, Ordering::SeqCst); + } +} diff --git a/compiler/rustc_codegen_cranelift/clean_all.sh b/compiler/rustc_codegen_cranelift/clean_all.sh new file mode 100755 index 00000000000..19405a53d1c --- /dev/null +++ b/compiler/rustc_codegen_cranelift/clean_all.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +rm -rf target/ build_system/target download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb + +# Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh +# FIXME remove at some point in the future +rm -rf rand/ regex/ simple-raytracer/ portable-simd/ abi-checker/ abi-cafe/ +rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version} diff --git a/compiler/rustc_codegen_cranelift/config.txt b/compiler/rustc_codegen_cranelift/config.txt new file mode 100644 index 00000000000..527ec5303b6 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/config.txt @@ -0,0 +1,54 @@ +# This file allows configuring the build system. + +# Which triple to produce a compiler toolchain for. +# +# Defaults to the default triple of rustc on the host system. +#host = x86_64-unknown-linux-gnu + +# Which triple to build libraries (core/alloc/std/test/proc_macro) for. +# +# Defaults to `host`. +#target = x86_64-unknown-linux-gnu + +# Disables cleaning of the sysroot dir. This will cause old compiled artifacts to be re-used when +# the sysroot source hasn't changed. This is useful when the codegen backend hasn't been modified. +# This option can be changed while the build system is already running for as long as sysroot +# building hasn't started yet. +#keep_sysroot + + +# Testsuite +# +# Each test suite item has a corresponding key here. The default is to run all tests. +# Comment any of these lines to skip individual tests. + +testsuite.no_sysroot +build.mini_core +build.example +jit.mini_core_hello_world +aot.mini_core_hello_world + +testsuite.base_sysroot +aot.arbitrary_self_types_pointers_and_wrappers +aot.issue_91827_extern_types +build.alloc_system +aot.alloc_example +jit.std_example +aot.std_example +aot.dst_field_align +aot.subslice-patterns-const-eval +aot.track-caller-attribute +aot.float-minmax-pass +aot.mod_bench +aot.issue-72793 +aot.issue-59326 +aot.polymorphize_coroutine +aot.neon +aot.gen_block_iterate +aot.raw-dylib + +testsuite.extended_sysroot +test.rust-random/rand +test.libcore +test.regex +test.portable-simd diff --git a/compiler/rustc_codegen_cranelift/docs/dwarf.md b/compiler/rustc_codegen_cranelift/docs/dwarf.md new file mode 100644 index 00000000000..502b1b03623 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/docs/dwarf.md @@ -0,0 +1,153 @@ +# Line number information + +Line number information maps between machine code instructions and the source level location. + +## Encoding + +The line number information is stored in the `.debug_line` section for ELF and `__debug_line` +section of the `__DWARF` segment for Mach-O object files. The line number information contains a +header followed by the line program. The line program is a program for a virtual machine with +instructions like set line number for the current machine code instruction and advance the current +machine code instruction. + +## Tips + +You need to set either `DW_AT_low_pc` and `DW_AT_high_pc` **or** `DW_AT_ranges` of a +`DW_TAG_compilation_unit` to the range of addresses in the compilation unit. After that you need +to set `DW_AT_stmt_list` to the `.debug_line` section offset of the line program. Otherwise a +debugger won't find the line number information. On macOS the debuginfo relocations **must** be +section relative and not symbol relative. +See [#303 (comment)](https://github.com/bjorn3/rustc_codegen_cranelift/issues/303#issuecomment-457825535) +for more information. + +# Function debuginfo + +## Tips + +`DW_TAG_subprogram` requires `DW_AT_name`, `DW_AT_low_pc` and `DW_AT_high_pc` **or** `DW_AT_ranges`. +Otherwise gdb will silently skip it. When `DW_AT_high_pc` is a length instead of an address, the +DWARF version must be at least 4. + +<details> +<summary>IRC log of #gdb on irc.freenode.org at 2020-04-23</summary> + +``` +(13:46:11) bjorn3: i am writing a backend for a compiler that uses DWARF for debuginfo. for some reason gdb seems to completely ignore all DW_TAG_subprogram, while lldb works fine. any idea what the problem could be? +(13:47:49) bjorn3: this is the output of llvm-dwarfdump: https://gist.github.com/bjorn3/8a34e333c80f13cb048381e94b4a3756 +(13:47:50) osa1: luispm: why is that problem not exists in 'commands'? (the target vs. host) +(13:52:16) luispm: osa1, commands is a bit more high level. It executes isolated commands. Breakpoint conditions need to be evaluated in the context of a valid expression. That expression may involve variables, symbols etc. +(13:52:36) luispm: osa1, Oh, i see your point now. Commands is only executed on the host. +(13:53:18) luispm: osa1, The commands are not tied to the execution context of the debugged program. The breakpoint conditions determine if execution must stop or continue etc. +(13:55:00) luispm: bjorn3, Likely something GDB thinks is wrong. Does enabling "set debug dwarf*" show anything? +(13:56:01) bjorn3: luispm: no +(13:56:12) bjorn3: for more context: https://github.com/bjorn3/rustc_codegen_cranelift/pull/978 +(13:58:16) osa1 verliet de ruimte (quit: Quit: osa1). +(13:58:28) bjorn3: luispm: wait, for b m<TAB> it shows nothing, but when stepping into a new function it does +(13:58:45) bjorn3: it still doesn't show anything for `info args` though +(13:58:50) bjorn3: No symbol table info available. +(14:00:50) luispm: bjorn3, Is that expected given the nature of the binary? +(14:01:17) bjorn3: b main<TAB> may show nothing as I only set DW_AT_linkage_name and not DW_AT_name +(14:01:24) bjorn3: info args should work though +(14:03:26) luispm: Sorry, I'm not sure what's up. There may be a genuine bug there. +(14:03:41) luispm: tromey (not currently in the channel, but maybe later today) may have more input. +(14:04:08) bjorn3: okay, thanks luispm! +(14:04:27) luispm: In the worst case, reporting a bug may prompt someone to look into that as well. +(14:04:48) luispm: Or send an e-mail to the gdb@sourceware.org mailing list. +(14:05:11) bjorn3: I don't know if it is a bug in gdb, or just me producing (slightly) wrong DWARF +(14:39:40) irker749: gdb: tom binutils-gdb.git:master * 740480b88af / gdb/ChangeLog gdb/darwin-nat.c gdb/inferior.c gdb/inferior.h: Remove iterate_over_inferiors +(15:22:45) irker749: gdb: tromey binutils-gdb.git:master * ecc6c6066b5 / gdb/ChangeLog gdb/dwarf2/read.c gdb/unittests/lookup_name_info-selftests.c: Fix Ada crash with .debug_names +(15:23:13) bjorn3: tromey: ping +(15:23:29) tromey: bjorn3: hey +(15:24:16) bjorn3: I am writing a backend for a compiler which uses DWARF for debuginfo. I unfortunately can't get gdb to show arguments. lldb works fine. +(15:25:13) bjorn3: it just says: No symbol table info available. +(15:25:21) bjorn3: any idea what it could be? +(15:25:34) bjorn3: dwarfdump output: https://gist.github.com/bjorn3/8a34e333c80f13cb048381e94b4a3756 +(15:26:48) bjorn3: more context: https://github.com/bjorn3/rustc_codegen_cranelift/pull/978 +(15:28:05) tromey: offhand I don't know, but if you can send me an executable I can look +(15:28:17) bjorn3: how should I send it? +(15:29:26) tromey: good question +(15:29:41) tromey: you could try emailing it to tromey at adacore.com +(15:29:47) tromey: dunno if that will work or not +(15:30:26) bjorn3: i will try +(15:37:27) bjorn3: tromey: i sent an email with the subject "gdb args not showing" +(15:38:29) tromey: will check now +(15:38:40) bjorn3: thanks! +(15:42:51) irker749: gdb: tdevries binutils-gdb.git:master * de82891ce5b / gdb/ChangeLog gdb/block.c gdb/block.h gdb/symtab.c gdb/testsuite/ChangeLog gdb/testsuite/gdb.base/decl-before-def-decl.c gdb/testsuite/gdb.base/decl-before-def-def.c gdb/testsuite/gdb.base/decl-before-def.exp: [gdb/symtab] Prefer def over decl (inter-CU case) +(15:42:52) irker749: gdb: tdevries binutils-gdb.git:master * 70bc38f5138 / gdb/ChangeLog gdb/symtab.c gdb/testsuite/ChangeLog gdb/testsuite/gdb.base/decl-before-def.exp: [gdb/symtab] Prefer def over decl (inter-CU case, with context) +(15:43:36) tromey: bjorn3: sorry, got distracted. I have the file now +(15:45:35) tromey: my first thing when investigating was to enable complaints +(15:45:37) tromey: so I did +(15:45:40) tromey: set complaints 1000 +(15:45:42) tromey: then +(15:45:51) tromey: file -readnow mini_core_hello_world +(15:46:00) tromey: gdb printed just one style of complaint +(15:46:07) tromey: During symbol reading: missing name for subprogram DIE at 0x3f7 +(15:46:18) tromey: (which is really pretty good, most compilers manage to generate a bunch) +(15:46:29) tromey: and then the gdb DWARF reader says +(15:46:34) tromey: /* Ignore functions with missing or empty names. These are actually +(15:46:34) tromey: illegal according to the DWARF standard. */ +(15:46:34) tromey: if (name == NULL) +(15:46:34) tromey: { +(15:46:37) tromey: complaint (_("missing name for subprogram DIE at %s"), +(15:46:40) tromey: sect_offset_str (die->sect_off)); +(15:46:47) tromey: I wonder if that comment is correct though +(15:47:34) tromey: I guess pedantically maybe it is, DWARF 5 3.3.1 says +(15:47:43) tromey: The subroutine or entry point entry has a DW_AT_name attribute whose value is +(15:47:43) tromey: a null-terminated string containing the subroutine or entry point name. +(15:48:14) bjorn3: i tried set complaints, but it returned complaints for system files. i didn't know about file -readnow. +(15:48:21) tromey: cool +(15:48:26) bjorn3: i will try adding DW_AT_name +(15:48:45) tromey: without readnow unfortunately you get less stuff, because for whatever reason gdb has 2 separate DWARF scanners +(15:49:02) tromey: sort of anyway +(15:49:43) tromey: this seems kind of pedantic of gdb, like if there's a linkage name but no DW_AT_name, then why bail? +(15:50:01) tromey: also what about anonymous functions +(15:50:17) tromey: but anyway this explains the current situation and if you don't mind adding DW_AT_name, then that's probably simplest +(15:51:47) bjorn3: i added DW_AT_name. +(15:51:54) bjorn3: now it says cannot get low and high bounds for subprogram DIE at ... +(15:52:01) tromey: ugh +(15:52:10) bjorn3: i will add DW_AT_low_pc and DW_AT_high_pc +(15:52:15) tromey: /* Ignore functions with missing or invalid low and high pc attributes. */ +(15:52:37) tromey: you can also use DW_AT_ranges +(15:52:55) tromey: if you'd prefer +(15:53:08) bjorn3: already using DW_AT_ranges for DW_TAG_compilation_unit +(15:53:19) bjorn3: for individual functions, there are no gaps +(15:57:07) bjorn3: still the same error with DW_AT_low_pc and DW_AT_high_pc +(15:57:24) bjorn3: tromey: ^ +(15:58:08) tromey: hmmm +(15:58:30) bjorn3: should i send the new executable? +(15:58:31) tromey: send me another executable & I will debug +(15:58:33) tromey: yep +(15:59:23) bjorn3: sent as repy of the previous mail +(16:03:23) tromey: the low PC has DW_FORM_addr, but the high PC has DW_FORM_udata, which seems weird +(16:03:50) mjw: no +(16:03:54) tromey: no? +(16:04:00) mjw: I suggested that for the DWARF standard... +(16:04:05) mjw: sorry +(16:04:58) mjw: The idea was that instead of two relocations and two address wide fields, you have one address and a constant offset. +(16:05:05) tromey: ahh, I see the code now +(16:05:07) tromey: I forgot about this +(16:05:18) tromey: if (cu->header.version >= 4 && attr_high->form_is_constant ()) +(16:05:18) tromey: high += low; +(16:05:36) mjw: that second offset doesn't need a relocation and can often be packed in something small, like an uleb128 +(16:05:51) mjw: using udata might not be ideal though, but is allowed +(16:05:51) tromey: bjorn3: the problem is that this CU claims to be DWARF 3 but is using a DWARF 4 feature +(16:05:58) mjw: aha +(16:05:59) bjorn3: which one? +(16:06:03) ryoshu: hi +(16:06:08) tromey: high_pc (udata) 107 (+0x00000000000011b0 <_ZN21mini_core_hello_world5start17hec55b7ca64fc434eE>) +(16:06:08) tromey: +(16:06:12) ryoshu: just soft ping, I have a queue of patches :) +(16:06:22) tromey: using this as a length requires DWARF 4 +(16:06:36) tromey: for gdb at least it's fine to always emit DWARF 4 +(16:06:44) bjorn3: trying dwarf 4 now +(16:06:48) tromey: I think there are some DWARF 5 features still in the works but DWARF 4 should be solid AFAIK +(16:07:03) tromey: fini +(16:07:08) tromey: lol wrong window +(16:07:56) mjw: Maybe you can accept it for DWARF < 4. But if I remember correctly it might be that people might have been using udata as if it was an address... +(16:08:13) tromey: yeah, I vaguely recall this as well, though I'd expect there to be a comment +(16:08:21) mjw: Cannot really remember why it needed version >= 4. Maybe there was no good reason? +(16:08:32) bjorn3: tromey: it works!!!! thanks for all the help! +(16:08:41) tromey: my pleasure bjorn3 +``` + +</details> diff --git a/compiler/rustc_codegen_cranelift/docs/rustc_testing.md b/compiler/rustc_codegen_cranelift/docs/rustc_testing.md new file mode 100644 index 00000000000..88c55557231 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/docs/rustc_testing.md @@ -0,0 +1,23 @@ +# Building and testing with changes in rustc code + +This is useful when changing code in `rustc_codegen_cranelift` as part of changing [main Rust repository](https://github.com/rust-lang/rust/). +This can happen, for example, when you are implementing a new compiler intrinsic. + +Instruction below uses `$RustCheckoutDir` as substitute for any folder where you cloned Rust repository. + +You need to do this steps to successfully compile and use the cranelift backend with your changes in rustc code: + +1. `cd $RustCheckoutDir` +2. Run `python x.py setup` and choose option for compiler (`b`). +3. Build compiler and necessary tools: `python x.py build --stage=2 compiler library/std src/tools/rustdoc src/tools/rustfmt` + * (Optional) You can also build cargo by adding `src/tools/cargo` to previous command. +4. Copy cargo from a nightly toolchain: `cp $(rustup +nightly which cargo) ./build/host/stage2/bin/cargo`. Note that you would need to do this every time you rebuilt `rust` repository. +5. Link your new `rustc` to toolchain: `rustup toolchain link stage2 ./build/host/stage2/`. +6. (Windows only) compile the build system: `rustc +stage2 -O build_system/main.rs -o y.exe`. +7. You need to prefix every `./y.sh` (or `y` if you built `build_system/main.rs` as `y`) command by `rustup run stage2` to make cg_clif use your local changes in rustc. + * `rustup run stage2 ./y.sh prepare` + * `rustup run stage2 ./y.sh build` + * (Optional) run tests: `rustup run stage2 ./y.sh test` +8. Now you can use your cg_clif build to compile other Rust programs, e.g. you can open any Rust crate and run commands like `$RustCheckoutDir/compiler/rustc_codegen_cranelift/dist/cargo-clif build --release`. + +You can also set `rust-analyzer.rustc.source` to your rust workspace to get rust-analyzer to understand your changes. diff --git a/compiler/rustc_codegen_cranelift/docs/usage.md b/compiler/rustc_codegen_cranelift/docs/usage.md new file mode 100644 index 00000000000..135a51ce392 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/docs/usage.md @@ -0,0 +1,67 @@ +# Usage + +rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. + +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.sh prepare` and `y.sh build` or `test.sh`). + +## Cargo + +In the directory with your project (where you can do the usual `cargo build`), run: + +```bash +$ $cg_clif_dir/dist/cargo-clif build +``` + +This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend. + +## Rustc + +> You should prefer using the Cargo method. + +```bash +$ $cg_clif_dir/dist/rustc-clif my_crate.rs +``` + +## Jit mode + +> ⚠⚠⚠ The JIT mode is highly experimental. It may be slower than AOT compilation due to lack of incremental compilation. It may also be hard to setup if you have cargo dependencies. ⚠⚠⚠ + +In jit mode cg_clif will immediately execute your code without creating an executable file. + +> This requires all dependencies to be available as dynamic library. +> The jit mode will probably need cargo integration to make this possible. + +```bash +$ $cg_clif_dir/dist/cargo-clif jit +``` + +or + +```bash +$ $cg_clif_dir/dist/rustc-clif -Zunstable-features -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs +``` + +There is also an experimental lazy jit mode. In this mode functions are only compiled once they are +first called. + +```bash +$ $cg_clif_dir/dist/cargo-clif lazy-jit +``` + +## Shell + +These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit. + +```bash +function jit_naked() { + echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-options -Cllvm-args=mode=jit-lazy -Cprefer-dynamic +} + +function jit() { + jit_naked "fn main() { $@ }" +} + +function jit_calc() { + jit 'println!("0x{:x}", ' $@ ');'; +} +``` diff --git a/compiler/rustc_codegen_cranelift/example/alloc_example.rs b/compiler/rustc_codegen_cranelift/example/alloc_example.rs new file mode 100644 index 00000000000..da70ca79439 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/alloc_example.rs @@ -0,0 +1,44 @@ +#![feature(start, core_intrinsics, alloc_error_handler, lang_items)] +#![allow(internal_features)] +#![no_std] + +extern crate alloc; +extern crate alloc_system; + +use alloc::boxed::Box; + +use alloc_system::System; + +#[global_allocator] +static ALLOC: System = System; + +#[cfg_attr(unix, link(name = "c"))] +#[cfg_attr(target_env = "msvc", link(name = "msvcrt"))] +extern "C" { + fn puts(s: *const u8) -> i32; +} + +#[panic_handler] +fn panic_handler(_: &core::panic::PanicInfo<'_>) -> ! { + core::intrinsics::abort(); +} + +#[alloc_error_handler] +fn alloc_error_handler(_: alloc::alloc::Layout) -> ! { + core::intrinsics::abort(); +} + +#[lang = "eh_personality"] +fn eh_personality() -> ! { + loop {} +} + +#[start] +fn main(_argc: isize, _argv: *const *const u8) -> isize { + let world: Box<&str> = Box::new("Hello World!\0"); + unsafe { + puts(*world as *const str as *const u8); + } + + 0 +} diff --git a/compiler/rustc_codegen_cranelift/example/alloc_system.rs b/compiler/rustc_codegen_cranelift/example/alloc_system.rs new file mode 100644 index 00000000000..2884c9c32ae --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/alloc_system.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: The Rust Project Developers (see https://thanks.rust-lang.org) + +#![no_std] + +pub struct System; + +#[cfg(any(windows, unix, target_os = "redox"))] +mod realloc_fallback { + use core::alloc::{GlobalAlloc, Layout}; + use core::{cmp, ptr}; + impl super::System { + pub(crate) unsafe fn realloc_fallback( + &self, + ptr: *mut u8, + old_layout: Layout, + new_size: usize, + ) -> *mut u8 { + // Docs for GlobalAlloc::realloc require this to be valid: + let new_layout = Layout::from_size_align_unchecked(new_size, old_layout.align()); + let new_ptr = GlobalAlloc::alloc(self, new_layout); + if !new_ptr.is_null() { + let size = cmp::min(old_layout.size(), new_size); + ptr::copy_nonoverlapping(ptr, new_ptr, size); + GlobalAlloc::dealloc(self, ptr, old_layout); + } + new_ptr + } + } +} +#[cfg(any(unix, target_os = "redox"))] +mod platform { + use core::alloc::{GlobalAlloc, Layout}; + use core::ffi::c_void; + use core::ptr; + + use System; + extern "C" { + fn posix_memalign(memptr: *mut *mut c_void, align: usize, size: usize) -> i32; + fn free(p: *mut c_void); + } + unsafe impl GlobalAlloc for System { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + aligned_malloc(&layout) + } + #[inline] + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let ptr = self.alloc(layout.clone()); + if !ptr.is_null() { + ptr::write_bytes(ptr, 0, layout.size()); + } + ptr + } + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + free(ptr as *mut c_void) + } + #[inline] + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + self.realloc_fallback(ptr, layout, new_size) + } + } + unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { + let mut out = ptr::null_mut(); + let ret = posix_memalign(&mut out, layout.align(), layout.size()); + if ret != 0 { ptr::null_mut() } else { out as *mut u8 } + } +} +#[cfg(windows)] +#[allow(nonstandard_style)] +mod platform { + use core::alloc::{GlobalAlloc, Layout}; + + use System; + type LPVOID = *mut u8; + type HANDLE = LPVOID; + type SIZE_T = usize; + type DWORD = u32; + type BOOL = i32; + extern "system" { + fn GetProcessHeap() -> HANDLE; + fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID; + fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID) -> BOOL; + fn GetLastError() -> DWORD; + } + #[repr(C)] + struct Header(*mut u8); + const HEAP_ZERO_MEMORY: DWORD = 0x00000008; + unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header { + &mut *(ptr as *mut Header).sub(1) + } + unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 { + let aligned = ptr.add(align - (ptr as usize & (align - 1))); + *get_header(aligned) = Header(ptr); + aligned + } + #[inline] + unsafe fn allocate_with_flags(layout: Layout, flags: DWORD) -> *mut u8 { + let size = layout.size() + layout.align(); + let ptr = HeapAlloc(GetProcessHeap(), flags, size); + (if ptr.is_null() { ptr } else { align_ptr(ptr, layout.align()) }) as *mut u8 + } + unsafe impl GlobalAlloc for System { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + allocate_with_flags(layout, 0) + } + #[inline] + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + allocate_with_flags(layout, HEAP_ZERO_MEMORY) + } + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + let header = get_header(ptr); + let err = HeapFree(GetProcessHeap(), 0, header.0 as LPVOID); + debug_assert!(err != 0, "Failed to free heap memory: {}", GetLastError()); + } + #[inline] + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + self.realloc_fallback(ptr, layout, new_size) + } + } +} diff --git a/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs b/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs new file mode 100644 index 00000000000..5479b0c617b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs @@ -0,0 +1,65 @@ +// Adapted from rustc run-pass test suite + +#![feature(arbitrary_self_types, unsize, coerce_unsized, dispatch_from_dyn)] + +use std::marker::Unsize; +use std::ops::{CoerceUnsized, Deref, DispatchFromDyn}; + +struct Ptr<T: ?Sized>(Box<T>); + +impl<T: ?Sized> Deref for Ptr<T> { + type Target = T; + + fn deref(&self) -> &T { + &*self.0 + } +} + +impl<T: Unsize<U> + ?Sized, U: ?Sized> CoerceUnsized<Ptr<U>> for Ptr<T> {} +impl<T: Unsize<U> + ?Sized, U: ?Sized> DispatchFromDyn<Ptr<U>> for Ptr<T> {} + +struct Wrapper<T: ?Sized>(T); + +impl<T: ?Sized> Deref for Wrapper<T> { + type Target = T; + + fn deref(&self) -> &T { + &self.0 + } +} + +impl<T: CoerceUnsized<U>, U> CoerceUnsized<Wrapper<U>> for Wrapper<T> {} +impl<T: DispatchFromDyn<U>, U> DispatchFromDyn<Wrapper<U>> for Wrapper<T> {} + +trait Trait { + // This method isn't object-safe yet. Unsized by-value `self` is object-safe (but not callable + // without unsized_locals), but wrappers around `Self` currently are not. + // FIXME (mikeyhew) uncomment this when unsized rvalues object-safety is implemented + // fn wrapper(self: Wrapper<Self>) -> i32; + fn ptr_wrapper(self: Ptr<Wrapper<Self>>) -> i32; + fn wrapper_ptr(self: Wrapper<Ptr<Self>>) -> i32; + fn wrapper_ptr_wrapper(self: Wrapper<Ptr<Wrapper<Self>>>) -> i32; +} + +impl Trait for i32 { + fn ptr_wrapper(self: Ptr<Wrapper<Self>>) -> i32 { + **self + } + fn wrapper_ptr(self: Wrapper<Ptr<Self>>) -> i32 { + **self + } + fn wrapper_ptr_wrapper(self: Wrapper<Ptr<Wrapper<Self>>>) -> i32 { + ***self + } +} + +fn main() { + let pw = Ptr(Box::new(Wrapper(5))) as Ptr<Wrapper<dyn Trait>>; + assert_eq!(pw.ptr_wrapper(), 5); + + let wp = Wrapper(Ptr(Box::new(6))) as Wrapper<Ptr<dyn Trait>>; + assert_eq!(wp.wrapper_ptr(), 6); + + let wpw = Wrapper(Ptr(Box::new(Wrapper(7)))) as Wrapper<Ptr<Wrapper<dyn Trait>>>; + assert_eq!(wpw.wrapper_ptr_wrapper(), 7); +} diff --git a/compiler/rustc_codegen_cranelift/example/dst-field-align.rs b/compiler/rustc_codegen_cranelift/example/dst-field-align.rs new file mode 100644 index 00000000000..22fc6ff33e3 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/dst-field-align.rs @@ -0,0 +1,66 @@ +// run-pass +#![allow(dead_code)] +struct Foo<T: ?Sized> { + a: u16, + b: T, +} + +trait Bar { + fn get(&self) -> usize; +} + +impl Bar for usize { + fn get(&self) -> usize { + *self + } +} + +struct Baz<T: ?Sized> { + a: T, +} + +struct HasDrop<T: ?Sized> { + ptr: Box<usize>, + data: T, +} + +fn main() { + // Test that zero-offset works properly + let b: Baz<usize> = Baz { a: 7 }; + assert_eq!(b.a.get(), 7); + let b: &Baz<dyn Bar> = &b; + assert_eq!(b.a.get(), 7); + + // Test that the field is aligned properly + let f: Foo<usize> = Foo { a: 0, b: 11 }; + assert_eq!(f.b.get(), 11); + let ptr1: *const u8 = &f.b as *const _ as *const u8; + + let f: &Foo<dyn Bar> = &f; + let ptr2: *const u8 = &f.b as *const _ as *const u8; + assert_eq!(f.b.get(), 11); + + // The pointers should be the same + assert_eq!(ptr1, ptr2); + + // Test that nested DSTs work properly + let f: Foo<Foo<usize>> = Foo { a: 0, b: Foo { a: 1, b: 17 } }; + assert_eq!(f.b.b.get(), 17); + let f: &Foo<Foo<dyn Bar>> = &f; + assert_eq!(f.b.b.get(), 17); + + // Test that get the pointer via destructuring works + + let f: Foo<usize> = Foo { a: 0, b: 11 }; + let f: &Foo<dyn Bar> = &f; + let &Foo { a: _, b: ref bar } = f; + assert_eq!(bar.get(), 11); + + // Make sure that drop flags don't screw things up + + let d: HasDrop<Baz<[i32; 4]>> = HasDrop { ptr: Box::new(0), data: Baz { a: [1, 2, 3, 4] } }; + assert_eq!([1, 2, 3, 4], d.data.a); + + let d: &HasDrop<Baz<[i32]>> = &d; + assert_eq!(&[1, 2, 3, 4], &d.data.a); +} diff --git a/compiler/rustc_codegen_cranelift/example/example.rs b/compiler/rustc_codegen_cranelift/example/example.rs new file mode 100644 index 00000000000..1ef2aa5dd8e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/example.rs @@ -0,0 +1,204 @@ +#![feature(no_core, unboxed_closures)] +#![no_core] +#![allow(dead_code)] + +extern crate mini_core; + +use mini_core::*; + +pub fn abc(a: u8) -> u8 { + a * 2 +} + +pub fn bcd(b: bool, a: u8) -> u8 { + if b { a * 2 } else { a * 3 } +} + +pub fn call() { + abc(42); +} + +pub fn indirect_call() { + let f: fn() = call; + f(); +} + +pub enum BoolOption { + Some(bool), + None, +} + +pub fn option_unwrap_or(o: BoolOption, d: bool) -> bool { + match o { + BoolOption::Some(b) => b, + BoolOption::None => d, + } +} + +pub fn ret_42() -> u8 { + 42 +} + +pub fn return_str() -> &'static str { + "hello world" +} + +pub fn promoted_val() -> &'static u8 { + &(1 * 2) +} + +pub fn cast_ref_to_raw_ptr(abc: &u8) -> *const u8 { + abc as *const u8 +} + +pub fn cmp_raw_ptr(a: *const u8, b: *const u8) -> bool { + a == b +} + +pub fn int_cast(a: u16, b: i16) -> (u8, u16, u32, usize, i8, i16, i32, isize, u8, u32) { + ( + a as u8, a as u16, a as u32, a as usize, a as i8, a as i16, a as i32, a as isize, b as u8, + b as u32, + ) +} + +pub fn char_cast(c: char) -> u8 { + c as u8 +} + +pub struct DebugTuple(()); + +pub fn debug_tuple() -> DebugTuple { + DebugTuple(()) +} + +pub fn size_of<T>() -> usize { + intrinsics::size_of::<T>() +} + +pub fn use_size_of() -> usize { + size_of::<u64>() +} + +pub unsafe fn use_copy_intrinsic(src: *const u8, dst: *mut u8) { + intrinsics::copy::<u8>(src, dst, 1); +} + +pub unsafe fn use_copy_intrinsic_ref(src: *const u8, dst: *mut u8) { + let copy2 = &intrinsics::copy::<u8>; + copy2(src, dst, 1); +} + +pub const ABC: u8 = 6 * 7; + +pub fn use_const() -> u8 { + ABC +} + +pub fn call_closure_3arg() { + (|_, _, _| {})(0u8, 42u16, 0u8) +} + +pub fn call_closure_2arg() { + (|_, _| {})(0u8, 42u16) +} + +pub struct IsNotEmpty; + +impl<'a, 'b> FnOnce<(&'a &'b [u16],)> for IsNotEmpty { + type Output = (u8, u8); + + #[inline] + extern "rust-call" fn call_once(mut self, arg: (&'a &'b [u16],)) -> (u8, u8) { + self.call_mut(arg) + } +} + +impl<'a, 'b> FnMut<(&'a &'b [u16],)> for IsNotEmpty { + #[inline] + extern "rust-call" fn call_mut(&mut self, _arg: (&'a &'b [u16],)) -> (u8, u8) { + (0, 42) + } +} + +pub fn call_is_not_empty() { + IsNotEmpty.call_once((&(&[0u16] as &[_]),)); +} + +pub fn eq_char(a: char, b: char) -> bool { + a == b +} + +pub unsafe fn transmute(c: char) -> u32 { + intrinsics::transmute(c) +} + +pub unsafe fn deref_str_ptr(s: *const str) -> &'static str { + &*s +} + +pub fn use_array(arr: [u8; 3]) -> u8 { + arr[1] +} + +pub fn repeat_array() -> [u8; 3] { + [0; 3] +} + +pub fn array_as_slice(arr: &[u8; 3]) -> &[u8] { + arr +} + +pub unsafe fn use_ctlz_nonzero(a: u16) -> u32 { + intrinsics::ctlz_nonzero(a) +} + +pub fn ptr_as_usize(ptr: *const u8) -> usize { + ptr as usize +} + +pub fn float_cast(a: f32, b: f64) -> (f64, f32) { + (a as f64, b as f32) +} + +pub fn int_to_float(a: u8, b: i32) -> (f64, f32) { + (a as f64, b as f32) +} + +pub fn make_array() -> [u8; 3] { + [42, 0, 5] +} + +pub fn some_promoted_tuple() -> &'static (&'static str, &'static str) { + &("abc", "some") +} + +pub fn index_slice(s: &[u8]) -> u8 { + s[2] +} + +pub struct StrWrapper { + s: str, +} + +pub fn str_wrapper_get(w: &StrWrapper) -> &str { + &w.s +} + +pub fn i16_as_i8(a: i16) -> i8 { + a as i8 +} + +pub struct Unsized(u8, str); + +pub fn get_sized_field_ref_from_unsized_type(u: &Unsized) -> &u8 { + &u.0 +} + +pub fn get_unsized_field_ref_from_unsized_type(u: &Unsized) -> &str { + &u.1 +} + +pub fn reuse_byref_argument_storage(a: (u8, u16, u32)) -> u8 { + a.0 +} diff --git a/compiler/rustc_codegen_cranelift/example/float-minmax-pass.rs b/compiler/rustc_codegen_cranelift/example/float-minmax-pass.rs new file mode 100644 index 00000000000..c54574d801d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/float-minmax-pass.rs @@ -0,0 +1,50 @@ +// Copied from https://github.com/rust-lang/rust/blob/3fe3b89cd57229343eeca753fdd8c63d9b03c65c/src/test/ui/simd/intrinsic/float-minmax-pass.rs +// run-pass +// ignore-emscripten + +// Test that the simd_f{min,max} intrinsics produce the correct results. + +#![feature(repr_simd, core_intrinsics)] +#![allow(internal_features, non_camel_case_types)] + +#[repr(simd)] +#[derive(Copy, Clone, PartialEq, Debug)] +struct f32x4(pub f32, pub f32, pub f32, pub f32); + +use std::intrinsics::simd::*; + +fn main() { + let x = f32x4(1.0, 2.0, 3.0, 4.0); + let y = f32x4(2.0, 1.0, 4.0, 3.0); + + #[cfg(not(any(target_arch = "mips", target_arch = "mips64")))] + let nan = f32::NAN; + // MIPS hardware except MIPS R6 treats f32::NAN as SNAN. Clear the signaling bit. + // See https://github.com/rust-lang/rust/issues/52746. + #[cfg(any(target_arch = "mips", target_arch = "mips64"))] + let nan = f32::from_bits(f32::NAN.to_bits() - 1); + + let n = f32x4(nan, nan, nan, nan); + + unsafe { + let min0 = simd_fmin(x, y); + let min1 = simd_fmin(y, x); + assert_eq!(min0, min1); + let e = f32x4(1.0, 1.0, 3.0, 3.0); + assert_eq!(min0, e); + let minn = simd_fmin(x, n); + assert_eq!(minn, x); + let minn = simd_fmin(y, n); + assert_eq!(minn, y); + + let max0 = simd_fmax(x, y); + let max1 = simd_fmax(y, x); + assert_eq!(max0, max1); + let e = f32x4(2.0, 2.0, 4.0, 4.0); + assert_eq!(max0, e); + let maxn = simd_fmax(x, n); + assert_eq!(maxn, x); + let maxn = simd_fmax(y, n); + assert_eq!(maxn, y); + } +} diff --git a/compiler/rustc_codegen_cranelift/example/gen_block_iterate.rs b/compiler/rustc_codegen_cranelift/example/gen_block_iterate.rs new file mode 100644 index 00000000000..25bfe542d22 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/gen_block_iterate.rs @@ -0,0 +1,36 @@ +// Copied from https://github.com/rust-lang/rust/blob/46455dc65069387f2dc46612f13fd45452ab301a/tests/ui/coroutine/gen_block_iterate.rs +// revisions: next old +//compile-flags: --edition 2024 -Zunstable-options +//[next] compile-flags: -Znext-solver +// run-pass +#![feature(gen_blocks)] + +fn foo() -> impl Iterator<Item = u32> { + gen { yield 42; for x in 3..6 { yield x } } +} + +fn moved() -> impl Iterator<Item = u32> { + let mut x = "foo".to_string(); + gen move { + yield 42; + if x == "foo" { return } + x.clear(); + for x in 3..6 { yield x } + } +} + +fn main() { + let mut iter = foo(); + assert_eq!(iter.next(), Some(42)); + assert_eq!(iter.next(), Some(3)); + assert_eq!(iter.next(), Some(4)); + assert_eq!(iter.next(), Some(5)); + assert_eq!(iter.next(), None); + // `gen` blocks are fused + assert_eq!(iter.next(), None); + + let mut iter = moved(); + assert_eq!(iter.next(), Some(42)); + assert_eq!(iter.next(), None); + +} diff --git a/compiler/rustc_codegen_cranelift/example/issue-59326.rs b/compiler/rustc_codegen_cranelift/example/issue-59326.rs new file mode 100644 index 00000000000..70b7c94e15c --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/issue-59326.rs @@ -0,0 +1,27 @@ +// Based on https://github.com/rust-lang/rust/blob/689511047a75a30825e367d4fd45c74604d0b15e/tests/ui/issues/issue-59326.rs#L1 +// check-pass +trait Service { + type S; +} + +trait Framing { + type F; +} + +impl Framing for () { + type F = (); +} + +trait HttpService<F: Framing>: Service<S = F::F> {} + +type BoxService = Box<dyn HttpService<(), S = ()>>; + +fn build_server<F: FnOnce() -> BoxService>(_: F) {} + +fn make_server<F: Framing>() -> Box<dyn HttpService<F, S = F::F>> { + unimplemented!() +} + +fn main() { + build_server(|| make_server()) +} diff --git a/compiler/rustc_codegen_cranelift/example/issue-72793.rs b/compiler/rustc_codegen_cranelift/example/issue-72793.rs new file mode 100644 index 00000000000..2e08fbca8ef --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/issue-72793.rs @@ -0,0 +1,29 @@ +// Adapted from rustc ui test suite (ui/type-alias-impl-trait/issue-72793.rs) + +#![feature(type_alias_impl_trait)] + +mod helper { + pub trait T { + type Item; + } + + pub type Alias<'a> = impl T<Item = &'a ()>; + + struct S; + impl<'a> T for &'a S { + type Item = &'a (); + } + + pub fn filter_positive<'a>() -> Alias<'a> { + &S + } +} +use helper::*; + +fn with_positive(fun: impl Fn(Alias<'_>)) { + fun(filter_positive()); +} + +fn main() { + with_positive(|_| ()); +} diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs new file mode 100644 index 00000000000..5e535ff62e1 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs @@ -0,0 +1,745 @@ +#![feature( + no_core, + lang_items, + intrinsics, + unboxed_closures, + extern_types, + decl_macro, + rustc_attrs, + transparent_unions, + auto_traits, + freeze_impls, + thread_local +)] +#![no_core] +#![allow(dead_code, internal_features, ambiguous_wide_pointer_comparisons)] + +#[lang = "sized"] +pub trait Sized {} + +#[lang = "destruct"] +pub trait Destruct {} + +#[lang = "tuple_trait"] +pub trait Tuple {} + +#[lang = "unsize"] +pub trait Unsize<T: ?Sized> {} + +#[lang = "coerce_unsized"] +pub trait CoerceUnsized<T> {} + +impl<'a, 'b: 'a, T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<&'a U> for &'b T {} +impl<'a, T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<&'a mut U> for &'a mut T {} +impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<*const U> for *const T {} +impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<*mut U> for *mut T {} + +#[lang = "dispatch_from_dyn"] +pub trait DispatchFromDyn<T> {} + +// &T -> &U +impl<'a, T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<&'a U> for &'a T {} +// &mut T -> &mut U +impl<'a, T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<&'a mut U> for &'a mut T {} +// *const T -> *const U +impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<*const U> for *const T {} +// *mut T -> *mut U +impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<*mut U> for *mut T {} +impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<Box<U>> for Box<T> {} + +#[lang = "receiver"] +pub trait Receiver {} + +impl<T: ?Sized> Receiver for &T {} +impl<T: ?Sized> Receiver for &mut T {} +impl<T: ?Sized> Receiver for Box<T> {} + +#[lang = "copy"] +pub unsafe trait Copy {} + +unsafe impl Copy for bool {} +unsafe impl Copy for u8 {} +unsafe impl Copy for u16 {} +unsafe impl Copy for u32 {} +unsafe impl Copy for u64 {} +unsafe impl Copy for u128 {} +unsafe impl Copy for usize {} +unsafe impl Copy for i8 {} +unsafe impl Copy for i16 {} +unsafe impl Copy for i32 {} +unsafe impl Copy for isize {} +unsafe impl Copy for f32 {} +unsafe impl Copy for f64 {} +unsafe impl Copy for char {} +unsafe impl<'a, T: ?Sized> Copy for &'a T {} +unsafe impl<T: ?Sized> Copy for *const T {} +unsafe impl<T: ?Sized> Copy for *mut T {} +unsafe impl<T: Copy> Copy for Option<T> {} + +#[lang = "sync"] +pub unsafe trait Sync {} + +unsafe impl Sync for bool {} +unsafe impl Sync for u8 {} +unsafe impl Sync for u16 {} +unsafe impl Sync for u32 {} +unsafe impl Sync for u64 {} +unsafe impl Sync for usize {} +unsafe impl Sync for i8 {} +unsafe impl Sync for i16 {} +unsafe impl Sync for i32 {} +unsafe impl Sync for isize {} +unsafe impl Sync for char {} +unsafe impl Sync for f32 {} +unsafe impl<'a, T: ?Sized> Sync for &'a T {} +unsafe impl<T: Sync, const N: usize> Sync for [T; N] {} + +#[lang = "freeze"] +unsafe auto trait Freeze {} + +unsafe impl<T: ?Sized> Freeze for PhantomData<T> {} +unsafe impl<T: ?Sized> Freeze for *const T {} +unsafe impl<T: ?Sized> Freeze for *mut T {} +unsafe impl<T: ?Sized> Freeze for &T {} +unsafe impl<T: ?Sized> Freeze for &mut T {} + +#[lang = "structural_peq"] +pub trait StructuralPartialEq {} + +#[lang = "not"] +pub trait Not { + type Output; + + fn not(self) -> Self::Output; +} + +impl Not for bool { + type Output = bool; + + fn not(self) -> bool { + !self + } +} + +#[lang = "mul"] +pub trait Mul<RHS = Self> { + type Output; + + #[must_use] + fn mul(self, rhs: RHS) -> Self::Output; +} + +impl Mul for u8 { + type Output = Self; + + fn mul(self, rhs: Self) -> Self::Output { + self * rhs + } +} + +impl Mul for usize { + type Output = Self; + + fn mul(self, rhs: Self) -> Self::Output { + self * rhs + } +} + +#[lang = "add"] +pub trait Add<RHS = Self> { + type Output; + + fn add(self, rhs: RHS) -> Self::Output; +} + +impl Add for u8 { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + self + rhs + } +} + +impl Add for i8 { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + self + rhs + } +} + +impl Add for usize { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + self + rhs + } +} + +#[lang = "sub"] +pub trait Sub<RHS = Self> { + type Output; + + fn sub(self, rhs: RHS) -> Self::Output; +} + +impl Sub for usize { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + self - rhs + } +} + +impl Sub for u8 { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + self - rhs + } +} + +impl Sub for i8 { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + self - rhs + } +} + +impl Sub for i16 { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + self - rhs + } +} + +#[lang = "rem"] +pub trait Rem<RHS = Self> { + type Output; + + fn rem(self, rhs: RHS) -> Self::Output; +} + +impl Rem for usize { + type Output = Self; + + fn rem(self, rhs: Self) -> Self { + self % rhs + } +} + +#[lang = "bitor"] +pub trait BitOr<RHS = Self> { + type Output; + + #[must_use] + fn bitor(self, rhs: RHS) -> Self::Output; +} + +impl BitOr for bool { + type Output = bool; + + fn bitor(self, rhs: bool) -> bool { + self | rhs + } +} + +impl<'a> BitOr<bool> for &'a bool { + type Output = bool; + + fn bitor(self, rhs: bool) -> bool { + *self | rhs + } +} + +#[lang = "eq"] +pub trait PartialEq<Rhs: ?Sized = Self> { + fn eq(&self, other: &Rhs) -> bool; + fn ne(&self, other: &Rhs) -> bool; +} + +impl PartialEq for u8 { + fn eq(&self, other: &u8) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &u8) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for u16 { + fn eq(&self, other: &u16) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &u16) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for u32 { + fn eq(&self, other: &u32) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &u32) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for u64 { + fn eq(&self, other: &u64) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &u64) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for u128 { + fn eq(&self, other: &u128) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &u128) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for usize { + fn eq(&self, other: &usize) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &usize) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for i8 { + fn eq(&self, other: &i8) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &i8) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for i32 { + fn eq(&self, other: &i32) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &i32) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for isize { + fn eq(&self, other: &isize) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &isize) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for char { + fn eq(&self, other: &char) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &char) -> bool { + (*self) != (*other) + } +} + +impl<T: ?Sized> PartialEq for *const T { + fn eq(&self, other: &*const T) -> bool { + *self == *other + } + fn ne(&self, other: &*const T) -> bool { + *self != *other + } +} + +impl<T: PartialEq> PartialEq for Option<T> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Some(lhs), Some(rhs)) => *lhs == *rhs, + (None, None) => true, + _ => false, + } + } + + fn ne(&self, other: &Self) -> bool { + match (self, other) { + (Some(lhs), Some(rhs)) => *lhs != *rhs, + (None, None) => false, + _ => true, + } + } +} + +#[lang = "shl"] +pub trait Shl<RHS = Self> { + type Output; + + #[must_use] + fn shl(self, rhs: RHS) -> Self::Output; +} + +impl Shl for u128 { + type Output = u128; + + fn shl(self, rhs: u128) -> u128 { + self << rhs + } +} + +#[lang = "neg"] +pub trait Neg { + type Output; + + fn neg(self) -> Self::Output; +} + +impl Neg for i8 { + type Output = i8; + + fn neg(self) -> i8 { + -self + } +} + +impl Neg for i16 { + type Output = i16; + + fn neg(self) -> i16 { + self + } +} + +impl Neg for isize { + type Output = isize; + + fn neg(self) -> isize { + -self + } +} + +impl Neg for f32 { + type Output = f32; + + fn neg(self) -> f32 { + -self + } +} + +pub enum Option<T> { + Some(T), + None, +} + +pub use Option::*; + +#[lang = "phantom_data"] +pub struct PhantomData<T: ?Sized>; + +#[lang = "fn_once"] +#[rustc_paren_sugar] +pub trait FnOnce<Args: Tuple> { + #[lang = "fn_once_output"] + type Output; + + extern "rust-call" fn call_once(self, args: Args) -> Self::Output; +} + +#[lang = "fn_mut"] +#[rustc_paren_sugar] +pub trait FnMut<Args: Tuple>: FnOnce<Args> { + extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output; +} + +#[lang = "panic"] +#[track_caller] +pub fn panic(_msg: &'static str) -> ! { + unsafe { + libc::puts("Panicking\n\0" as *const str as *const i8); + intrinsics::abort(); + } +} + +macro_rules! panic_const { + ($($lang:ident = $message:expr,)+) => { + pub mod panic_const { + use super::*; + + $( + #[track_caller] + #[lang = stringify!($lang)] + pub fn $lang() -> ! { + panic($message); + } + )+ + } + } +} + +panic_const! { + panic_const_add_overflow = "attempt to add with overflow", + panic_const_sub_overflow = "attempt to subtract with overflow", + panic_const_mul_overflow = "attempt to multiply with overflow", + panic_const_div_overflow = "attempt to divide with overflow", + panic_const_rem_overflow = "attempt to calculate the remainder with overflow", + panic_const_neg_overflow = "attempt to negate with overflow", + panic_const_shr_overflow = "attempt to shift right with overflow", + panic_const_shl_overflow = "attempt to shift left with overflow", + panic_const_div_by_zero = "attempt to divide by zero", + panic_const_rem_by_zero = "attempt to calculate the remainder with a divisor of zero", +} + +#[lang = "panic_bounds_check"] +#[track_caller] +fn panic_bounds_check(index: usize, len: usize) -> ! { + unsafe { + libc::printf( + "index out of bounds: the len is %d but the index is %d\n\0" as *const str as *const i8, + len, + index, + ); + intrinsics::abort(); + } +} + +#[lang = "panic_cannot_unwind"] +#[track_caller] +fn panic_cannot_unwind() -> ! { + unsafe { + libc::puts("panic in a function that cannot unwind\n\0" as *const str as *const i8); + intrinsics::abort(); + } +} + +#[lang = "eh_personality"] +fn eh_personality() -> ! { + loop {} +} + +#[lang = "drop_in_place"] +#[allow(unconditional_recursion)] +pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) { + // Code here does not matter - this is replaced by the + // real drop glue by the compiler. + drop_in_place(to_drop); +} + +#[lang = "unpin"] +pub auto trait Unpin {} + +#[lang = "deref"] +pub trait Deref { + type Target: ?Sized; + + fn deref(&self) -> &Self::Target; +} + +#[repr(transparent)] +#[rustc_layout_scalar_valid_range_start(1)] +#[rustc_nonnull_optimization_guaranteed] +pub struct NonNull<T: ?Sized>(pub *const T); + +impl<T: ?Sized, U: ?Sized> CoerceUnsized<NonNull<U>> for NonNull<T> where T: Unsize<U> {} +impl<T: ?Sized, U: ?Sized> DispatchFromDyn<NonNull<U>> for NonNull<T> where T: Unsize<U> {} + +pub struct Unique<T: ?Sized> { + pub pointer: NonNull<T>, + pub _marker: PhantomData<T>, +} + +impl<T: ?Sized, U: ?Sized> CoerceUnsized<Unique<U>> for Unique<T> where T: Unsize<U> {} +impl<T: ?Sized, U: ?Sized> DispatchFromDyn<Unique<U>> for Unique<T> where T: Unsize<U> {} + +#[lang = "global_alloc_ty"] +pub struct Global; + +#[lang = "owned_box"] +pub struct Box<T: ?Sized, A = Global>(Unique<T>, A); + +impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<Box<U>> for Box<T> {} + +impl<T> Box<T> { + pub fn new(val: T) -> Box<T> { + unsafe { + let size = intrinsics::size_of::<T>(); + let ptr = libc::malloc(size); + intrinsics::copy(&val as *const T as *const u8, ptr, size); + Box(Unique { pointer: NonNull(ptr as *const T), _marker: PhantomData }, Global) + } + } +} + +impl<T: ?Sized, A> Drop for Box<T, A> { + fn drop(&mut self) { + // inner value is dropped by compiler + unsafe { + libc::free(self.0.pointer.0 as *mut u8); + } + } +} + +impl<T: ?Sized> Deref for Box<T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &**self + } +} + +#[lang = "exchange_malloc"] +unsafe fn allocate(size: usize, _align: usize) -> *mut u8 { + libc::malloc(size) +} + +#[lang = "drop"] +pub trait Drop { + fn drop(&mut self); +} + +#[lang = "manually_drop"] +#[repr(transparent)] +pub struct ManuallyDrop<T: ?Sized> { + pub value: T, +} + +#[lang = "maybe_uninit"] +#[repr(transparent)] +pub union MaybeUninit<T> { + pub uninit: (), + pub value: ManuallyDrop<T>, +} + +pub mod intrinsics { + extern "rust-intrinsic" { + #[rustc_safe_intrinsic] + pub fn abort() -> !; + #[rustc_safe_intrinsic] + pub fn size_of<T>() -> usize; + pub fn size_of_val<T: ?::Sized>(val: *const T) -> usize; + #[rustc_safe_intrinsic] + pub fn min_align_of<T>() -> usize; + pub fn min_align_of_val<T: ?::Sized>(val: *const T) -> usize; + pub fn copy<T>(src: *const T, dst: *mut T, count: usize); + pub fn transmute<T, U>(e: T) -> U; + pub fn ctlz_nonzero<T>(x: T) -> u32; + #[rustc_safe_intrinsic] + pub fn needs_drop<T: ?::Sized>() -> bool; + #[rustc_safe_intrinsic] + pub fn bitreverse<T>(x: T) -> T; + #[rustc_safe_intrinsic] + pub fn bswap<T>(x: T) -> T; + pub fn write_bytes<T>(dst: *mut T, val: u8, count: usize); + } +} + +pub mod libc { + // With the new Universal CRT, msvc has switched to all the printf functions being inline wrapper + // functions. legacy_stdio_definitions.lib which provides the printf wrapper functions as normal + // symbols to link against. + #[cfg_attr(unix, link(name = "c"))] + #[cfg_attr(target_env = "msvc", link(name = "legacy_stdio_definitions"))] + extern "C" { + pub fn printf(format: *const i8, ...) -> i32; + } + + #[cfg_attr(unix, link(name = "c"))] + #[cfg_attr(target_env = "msvc", link(name = "msvcrt"))] + extern "C" { + pub fn puts(s: *const i8) -> i32; + pub fn malloc(size: usize) -> *mut u8; + pub fn free(ptr: *mut u8); + pub fn memcpy(dst: *mut u8, src: *const u8, size: usize); + pub fn memmove(dst: *mut u8, src: *const u8, size: usize); + pub fn strncpy(dst: *mut u8, src: *const u8, size: usize); + } +} + +#[lang = "index"] +pub trait Index<Idx: ?Sized> { + type Output: ?Sized; + fn index(&self, index: Idx) -> &Self::Output; +} + +impl<T> Index<usize> for [T; 3] { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + &self[index] + } +} + +impl<T> Index<usize> for [T] { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + &self[index] + } +} + +extern "C" { + type VaListImpl; +} + +#[lang = "va_list"] +#[repr(transparent)] +pub struct VaList<'a>(&'a mut VaListImpl); + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro stringify($($t:tt)*) { + /* compiler built-in */ +} + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro file() { + /* compiler built-in */ +} + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro line() { + /* compiler built-in */ +} + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro cfg() { + /* compiler built-in */ +} + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro asm() { + /* compiler built-in */ +} + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro global_asm() { + /* compiler built-in */ +} + +pub static A_STATIC: u8 = 42; + +#[lang = "panic_location"] +struct PanicLocation { + file: &'static str, + line: u32, + column: u32, +} + +#[no_mangle] +#[cfg(not(all(windows, target_env = "gnu")))] +pub fn get_tls() -> u8 { + #[thread_local] + static A: u8 = 42; + + A +} diff --git a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs new file mode 100644 index 00000000000..7d361a9ab2b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs @@ -0,0 +1,616 @@ +#![feature( + no_core, + lang_items, + never_type, + linkage, + extern_types, + naked_functions, + thread_local, + repr_simd, + raw_ref_op +)] +#![no_core] +#![allow(dead_code, non_camel_case_types, internal_features)] + +extern crate mini_core; + +use mini_core::libc::*; +use mini_core::*; + +macro_rules! assert { + ($e:expr) => { + if !$e { + panic(stringify!(!$e)); + } + }; +} + +macro_rules! assert_eq { + ($l:expr, $r: expr) => { + if $l != $r { + panic(stringify!($l != $r)); + } + }; +} + +#[lang = "termination"] +trait Termination { + fn report(self) -> i32; +} + +impl Termination for () { + fn report(self) -> i32 { + unsafe { + NUM = 6 * 7 + 1 + (1u8 == 1u8) as u8; // 44 + assert_eq!(*NUM_REF as i32, 44); + } + 0 + } +} + +trait SomeTrait { + fn object_safe(&self); +} + +impl SomeTrait for &'static str { + fn object_safe(&self) { + unsafe { + puts(*self as *const str as *const i8); + } + } +} + +struct NoisyDrop { + text: &'static str, + inner: NoisyDropInner, +} + +struct NoisyDropUnsized { + inner: NoisyDropInner, + text: str, +} + +struct NoisyDropInner; + +impl Drop for NoisyDrop { + fn drop(&mut self) { + unsafe { + puts(self.text as *const str as *const i8); + } + } +} + +impl Drop for NoisyDropInner { + fn drop(&mut self) { + unsafe { + puts("Inner got dropped!\0" as *const str as *const i8); + } + } +} + +impl SomeTrait for NoisyDrop { + fn object_safe(&self) {} +} + +enum Ordering { + Less = -1, + Equal = 0, + Greater = 1, +} + +#[lang = "start"] +fn start<T: Termination + 'static>( + main: fn() -> T, + argc: isize, + argv: *const *const u8, + _sigpipe: u8, +) -> isize { + if argc == 3 { + unsafe { + puts(*argv as *const i8); + } + unsafe { + puts(*((argv as usize + intrinsics::size_of::<*const u8>()) as *const *const i8)); + } + unsafe { + puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8)); + } + } + + main().report() as isize +} + +static mut NUM: u8 = 6 * 7; + +static NUM_REF: &'static u8 = unsafe { &*&raw const NUM }; + +unsafe fn zeroed<T>() -> T { + let mut uninit = MaybeUninit { uninit: () }; + intrinsics::write_bytes(&mut uninit.value.value as *mut T, 0, 1); + uninit.value.value +} + +fn take_f32(_f: f32) {} +fn take_unique(_u: Unique<()>) {} + +fn return_u128_pair() -> (u128, u128) { + (0, 0) +} + +fn call_return_u128_pair() { + return_u128_pair(); +} + +#[repr(C)] +pub struct bool_11 { + field0: bool, + field1: bool, + field2: bool, + field3: bool, + field4: bool, + field5: bool, + field6: bool, + field7: bool, + field8: bool, + field9: bool, + field10: bool, +} + +extern "C" fn bool_struct_in_11(_arg0: bool_11) {} + +#[allow(unreachable_code)] // FIXME false positive +fn main() { + take_unique(Unique { pointer: unsafe { NonNull(1 as *mut ()) }, _marker: PhantomData }); + take_f32(0.1); + + call_return_u128_pair(); + + bool_struct_in_11(bool_11 { + field0: true, + field1: true, + field2: true, + field3: true, + field4: true, + field5: true, + field6: true, + field7: true, + field8: true, + field9: true, + field10: true, + }); + + let slice = &[0, 1] as &[i32]; + let slice_ptr = slice as *const [i32] as *const i32; + + assert_eq!(slice_ptr as usize % 4, 0); + + unsafe { + printf("Hello %s\n\0" as *const str as *const i8, "printf\0" as *const str as *const i8); + + let hello: &[u8] = b"Hello\0" as &[u8; 6]; + let ptr: *const i8 = hello as *const [u8] as *const i8; + puts(ptr); + + let world: Box<&str> = Box::new("World!\0"); + puts(*world as *const str as *const i8); + world as Box<dyn SomeTrait>; + + assert_eq!(intrinsics::bitreverse(0b10101000u8), 0b00010101u8); + + assert_eq!(intrinsics::bswap(0xabu8), 0xabu8); + assert_eq!(intrinsics::bswap(0xddccu16), 0xccddu16); + assert_eq!(intrinsics::bswap(0xffee_ddccu32), 0xccdd_eeffu32); + assert_eq!(intrinsics::bswap(0x1234_5678_ffee_ddccu64), 0xccdd_eeff_7856_3412u64); + + assert_eq!(intrinsics::size_of_val(hello) as u8, 6); + + let chars = &['C', 'h', 'a', 'r', 's']; + let chars = chars as &[char]; + assert_eq!(intrinsics::size_of_val(chars) as u8, 4 * 5); + + let a: &dyn SomeTrait = &"abc\0"; + a.object_safe(); + + assert_eq!(intrinsics::size_of_val(a) as u8, 16); + assert_eq!(intrinsics::size_of_val(&0u32) as u8, 4); + + assert_eq!(intrinsics::min_align_of::<u16>() as u8, 2); + assert_eq!( + intrinsics::min_align_of_val(&a) as u8, + intrinsics::min_align_of::<&str>() as u8 + ); + + assert!(!intrinsics::needs_drop::<u8>()); + assert!(!intrinsics::needs_drop::<[u8]>()); + assert!(intrinsics::needs_drop::<NoisyDrop>()); + assert!(intrinsics::needs_drop::<NoisyDropUnsized>()); + + Unique { pointer: NonNull(1 as *mut &str), _marker: PhantomData } as Unique<dyn SomeTrait>; + + struct MyDst<T: ?Sized>(T); + + intrinsics::size_of_val(&MyDst([0u8; 4]) as &MyDst<[u8]>); + + struct Foo { + x: u8, + y: !, + } + + unsafe fn uninitialized<T>() -> T { + MaybeUninit { uninit: () }.value.value + } + + zeroed::<(u8, u8)>(); + #[allow(unreachable_code)] + { + if false { + zeroed::<!>(); + zeroed::<Foo>(); + uninitialized::<Foo>(); + } + } + } + + let _ = Box::new(NoisyDrop { text: "Boxed outer got dropped!\0", inner: NoisyDropInner }) + as Box<dyn SomeTrait>; + + const FUNC_REF: Option<fn()> = Some(main); + match FUNC_REF { + Some(_) => {} + None => assert!(false), + } + + match Ordering::Less { + Ordering::Less => {} + _ => assert!(false), + } + + [NoisyDropInner, NoisyDropInner]; + + let x = &[0u32, 42u32] as &[u32]; + match x { + [] => assert_eq!(0u32, 1), + [_, ref y @ ..] => assert_eq!(&x[1] as *const u32 as usize, &y[0] as *const u32 as usize), + } + + assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42); + + #[cfg(not(any(jit, windows)))] + { + extern "C" { + #[linkage = "extern_weak"] + static ABC: *const u8; + } + + { + extern "C" { + #[linkage = "extern_weak"] + static ABC: *const u8; + } + } + + unsafe { + assert_eq!(ABC as usize, 0); + } + } + + &mut (|| Some(0 as *const ())) as &mut dyn FnMut() -> Option<*const ()>; + + let f = 1000.0; + assert_eq!(f as u8, 255); + let f2 = -1000.0; + assert_eq!(f2 as i8, -128); + assert_eq!(f2 as u8, 0); + + let amount = 0; + assert_eq!(1u128 << amount, 1); + + static ANOTHER_STATIC: &u8 = &A_STATIC; + assert_eq!(*ANOTHER_STATIC, 42); + + check_niche_behavior(); + + extern "C" { + type ExternType; + } + + struct ExternTypeWrapper { + _a: ExternType, + } + + let nullptr = 0 as *const (); + let extern_nullptr = nullptr as *const ExternTypeWrapper; + extern_nullptr as *const (); + let slice_ptr = &[] as *const [u8]; + slice_ptr as *const u8; + + let repeat = [Some(42); 2]; + assert_eq!(repeat[0], Some(42)); + assert_eq!(repeat[1], Some(42)); + + from_decimal_string(); + + #[cfg(all(not(jit), not(all(windows, target_env = "gnu"))))] + test_tls(); + + #[cfg(all(not(jit), target_arch = "x86_64", any(target_os = "linux", target_os = "macos")))] + unsafe { + global_asm_test(); + naked_test(); + } + + // Both statics have a reference that points to the same anonymous allocation. + static REF1: &u8 = &42; + static REF2: &u8 = REF1; + assert_eq!(*REF1, *REF2); + + #[repr(simd)] + struct V([f64; 2]); + + let f = V([0.0, 1.0]); + let _a = f.0[0]; + + stack_val_align(); +} + +#[inline(never)] +fn stack_val_align() { + #[repr(align(8192))] + struct Foo(u8); + + let a = Foo(0); + assert_eq!(&a as *const Foo as usize % 8192, 0); +} + +#[cfg(all(not(jit), target_arch = "x86_64", any(target_os = "linux", target_os = "macos")))] +extern "C" { + fn global_asm_test(); +} + +#[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))] +global_asm! { + " + .global global_asm_test + global_asm_test: + // comment that would normally be removed by LLVM + ret + " +} + +#[cfg(all(not(jit), target_arch = "x86_64", target_os = "macos"))] +global_asm! { + " + .global _global_asm_test + _global_asm_test: + // comment that would normally be removed by LLVM + ret + " +} + +#[cfg(all(not(jit), target_arch = "x86_64"))] +#[naked] +extern "C" fn naked_test() { + unsafe { + asm!("ret", options(noreturn)); + } +} + +#[repr(C)] +enum c_void { + _1, + _2, +} + +type c_int = i32; +type c_ulong = u64; + +type pthread_t = c_ulong; + +#[repr(C)] +struct pthread_attr_t { + __size: [u64; 7], +} + +#[link(name = "pthread")] +#[cfg(unix)] +extern "C" { + fn pthread_attr_init(attr: *mut pthread_attr_t) -> c_int; + + fn pthread_create( + native: *mut pthread_t, + attr: *const pthread_attr_t, + f: extern "C" fn(_: *mut c_void) -> *mut c_void, + value: *mut c_void, + ) -> c_int; + + fn pthread_join(native: pthread_t, value: *mut *mut c_void) -> c_int; +} + +type DWORD = u32; +type LPDWORD = *mut u32; + +type LPVOID = *mut c_void; +type HANDLE = *mut c_void; + +#[link(name = "msvcrt")] +#[cfg(windows)] +extern "C" { + fn WaitForSingleObject(hHandle: LPVOID, dwMilliseconds: DWORD) -> DWORD; + + fn CreateThread( + lpThreadAttributes: LPVOID, // Technically LPSECURITY_ATTRIBUTES, but we don't use it anyway + dwStackSize: usize, + lpStartAddress: extern "C" fn(_: *mut c_void) -> *mut c_void, + lpParameter: LPVOID, + dwCreationFlags: DWORD, + lpThreadId: LPDWORD, + ) -> HANDLE; +} + +struct Thread { + #[cfg(windows)] + handle: HANDLE, + #[cfg(unix)] + handle: pthread_t, +} + +impl Thread { + unsafe fn create(f: extern "C" fn(_: *mut c_void) -> *mut c_void) -> Self { + #[cfg(unix)] + { + let mut attr: pthread_attr_t = zeroed(); + let mut thread: pthread_t = 0; + + if pthread_attr_init(&mut attr) != 0 { + assert!(false); + } + + if pthread_create(&mut thread, &attr, f, 0 as *mut c_void) != 0 { + assert!(false); + } + + Thread { handle: thread } + } + + #[cfg(windows)] + { + let handle = CreateThread(0 as *mut c_void, 0, f, 0 as *mut c_void, 0, 0 as *mut u32); + + if (handle as u64) == 0 { + assert!(false); + } + + Thread { handle } + } + } + + unsafe fn join(self) { + #[cfg(unix)] + { + let mut res = 0 as *mut c_void; + pthread_join(self.handle, &mut res); + } + + #[cfg(windows)] + { + // The INFINITE macro is used to signal operations that do not timeout. + let infinite = 0xffffffff; + assert!(WaitForSingleObject(self.handle, infinite) == 0); + } + } +} + +#[thread_local] +#[cfg(not(jit))] +static mut TLS: u8 = 42; + +#[cfg(not(jit))] +extern "C" fn mutate_tls(_: *mut c_void) -> *mut c_void { + unsafe { + TLS = 0; + } + 0 as *mut c_void +} + +#[cfg(not(jit))] +fn test_tls() { + unsafe { + assert_eq!(TLS, 42); + + let thread = Thread::create(mutate_tls); + thread.join(); + + // TLS of main thread must not have been changed by the other thread. + assert_eq!(TLS, 42); + + puts("TLS works!\n\0" as *const str as *const i8); + } +} + +// Copied ui/issues/issue-61696.rs + +pub enum Infallible {} + +// The check that the `bool` field of `V1` is encoding a "niche variant" +// (i.e. not `V1`, so `V3` or `V4`) used to be mathematically incorrect, +// causing valid `V1` values to be interpreted as other variants. +pub enum E1 { + V1 { f: bool }, + V2 { f: Infallible }, + V3, + V4, +} + +// Computing the discriminant used to be done using the niche type (here `u8`, +// from the `bool` field of `V1`), overflowing for variants with large enough +// indices (`V3` and `V4`), causing them to be interpreted as other variants. +#[rustfmt::skip] +pub enum E2<X> { + V1 { f: bool }, + + /*_00*/ _01(X), _02(X), _03(X), _04(X), _05(X), _06(X), _07(X), + _08(X), _09(X), _0A(X), _0B(X), _0C(X), _0D(X), _0E(X), _0F(X), + _10(X), _11(X), _12(X), _13(X), _14(X), _15(X), _16(X), _17(X), + _18(X), _19(X), _1A(X), _1B(X), _1C(X), _1D(X), _1E(X), _1F(X), + _20(X), _21(X), _22(X), _23(X), _24(X), _25(X), _26(X), _27(X), + _28(X), _29(X), _2A(X), _2B(X), _2C(X), _2D(X), _2E(X), _2F(X), + _30(X), _31(X), _32(X), _33(X), _34(X), _35(X), _36(X), _37(X), + _38(X), _39(X), _3A(X), _3B(X), _3C(X), _3D(X), _3E(X), _3F(X), + _40(X), _41(X), _42(X), _43(X), _44(X), _45(X), _46(X), _47(X), + _48(X), _49(X), _4A(X), _4B(X), _4C(X), _4D(X), _4E(X), _4F(X), + _50(X), _51(X), _52(X), _53(X), _54(X), _55(X), _56(X), _57(X), + _58(X), _59(X), _5A(X), _5B(X), _5C(X), _5D(X), _5E(X), _5F(X), + _60(X), _61(X), _62(X), _63(X), _64(X), _65(X), _66(X), _67(X), + _68(X), _69(X), _6A(X), _6B(X), _6C(X), _6D(X), _6E(X), _6F(X), + _70(X), _71(X), _72(X), _73(X), _74(X), _75(X), _76(X), _77(X), + _78(X), _79(X), _7A(X), _7B(X), _7C(X), _7D(X), _7E(X), _7F(X), + _80(X), _81(X), _82(X), _83(X), _84(X), _85(X), _86(X), _87(X), + _88(X), _89(X), _8A(X), _8B(X), _8C(X), _8D(X), _8E(X), _8F(X), + _90(X), _91(X), _92(X), _93(X), _94(X), _95(X), _96(X), _97(X), + _98(X), _99(X), _9A(X), _9B(X), _9C(X), _9D(X), _9E(X), _9F(X), + _A0(X), _A1(X), _A2(X), _A3(X), _A4(X), _A5(X), _A6(X), _A7(X), + _A8(X), _A9(X), _AA(X), _AB(X), _AC(X), _AD(X), _AE(X), _AF(X), + _B0(X), _B1(X), _B2(X), _B3(X), _B4(X), _B5(X), _B6(X), _B7(X), + _B8(X), _B9(X), _BA(X), _BB(X), _BC(X), _BD(X), _BE(X), _BF(X), + _C0(X), _C1(X), _C2(X), _C3(X), _C4(X), _C5(X), _C6(X), _C7(X), + _C8(X), _C9(X), _CA(X), _CB(X), _CC(X), _CD(X), _CE(X), _CF(X), + _D0(X), _D1(X), _D2(X), _D3(X), _D4(X), _D5(X), _D6(X), _D7(X), + _D8(X), _D9(X), _DA(X), _DB(X), _DC(X), _DD(X), _DE(X), _DF(X), + _E0(X), _E1(X), _E2(X), _E3(X), _E4(X), _E5(X), _E6(X), _E7(X), + _E8(X), _E9(X), _EA(X), _EB(X), _EC(X), _ED(X), _EE(X), _EF(X), + _F0(X), _F1(X), _F2(X), _F3(X), _F4(X), _F5(X), _F6(X), _F7(X), + _F8(X), _F9(X), _FA(X), _FB(X), _FC(X), _FD(X), _FE(X), _FF(X), + + V3, + V4, +} + +fn check_niche_behavior() { + if let E1::V2 { .. } = (E1::V1 { f: true }) { + intrinsics::abort(); + } + + if let E2::V1 { .. } = E2::V3::<Infallible> { + intrinsics::abort(); + } +} + +fn from_decimal_string() { + loop { + let multiplier = 1; + + take_multiplier_ref(&multiplier); + + if multiplier == 1 { + break; + } + + unreachable(); + } +} + +fn take_multiplier_ref(_multiplier: &u128) {} + +fn unreachable() -> ! { + panic("unreachable") +} diff --git a/compiler/rustc_codegen_cranelift/example/mod_bench.rs b/compiler/rustc_codegen_cranelift/example/mod_bench.rs new file mode 100644 index 00000000000..11a3e8fc72d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/mod_bench.rs @@ -0,0 +1,37 @@ +#![feature(start, core_intrinsics, lang_items)] +#![allow(internal_features)] +#![no_std] + +#[cfg_attr(unix, link(name = "c"))] +#[cfg_attr(target_env = "msvc", link(name = "msvcrt"))] +extern "C" {} + +#[panic_handler] +fn panic_handler(_: &core::panic::PanicInfo<'_>) -> ! { + core::intrinsics::abort(); +} + +#[lang = "eh_personality"] +fn eh_personality() {} + +// Required for rustc_codegen_llvm +#[no_mangle] +unsafe extern "C" fn _Unwind_Resume() { + core::intrinsics::unreachable(); +} + +#[start] +fn main(_argc: isize, _argv: *const *const u8) -> isize { + for i in 2..10_000_000 { + black_box((i + 1) % i); + } + + 0 +} + +#[inline(never)] +fn black_box(i: u32) { + if i != 1 { + core::intrinsics::abort(); + } +} diff --git a/compiler/rustc_codegen_cranelift/example/neon.rs b/compiler/rustc_codegen_cranelift/example/neon.rs new file mode 100644 index 00000000000..00e437d7e54 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/neon.rs @@ -0,0 +1,236 @@ +// Most of these tests are copied from https://github.com/japaric/stdsimd/blob/0f4413d01c4f0c3ffbc5a69e9a37fbc7235b31a9/coresimd/arm/neon.rs + +#![feature(portable_simd)] + +#[cfg(target_arch = "aarch64")] +use std::arch::aarch64::*; +#[cfg(target_arch = "aarch64")] +use std::mem::transmute; +#[cfg(target_arch = "aarch64")] +use std::simd::*; + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmin_s8() { + let a = i8x8::from([1, -2, 3, -4, 5, 6, 7, 8]); + let b = i8x8::from([0, 3, 2, 5, 4, 7, 6, 9]); + let e = i8x8::from([-2, -4, 5, 7, 0, 2, 4, 6]); + let r: i8x8 = transmute(vpmin_s8(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmin_s16() { + let a = i16x4::from([1, 2, 3, -4]); + let b = i16x4::from([0, 3, 2, 5]); + let e = i16x4::from([1, -4, 0, 2]); + let r: i16x4 = transmute(vpmin_s16(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmin_s32() { + let a = i32x2::from([1, -2]); + let b = i32x2::from([0, 3]); + let e = i32x2::from([-2, 0]); + let r: i32x2 = transmute(vpmin_s32(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmin_u8() { + let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 8]); + let b = u8x8::from([0, 3, 2, 5, 4, 7, 6, 9]); + let e = u8x8::from([1, 3, 5, 7, 0, 2, 4, 6]); + let r: u8x8 = transmute(vpmin_u8(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmin_u16() { + let a = u16x4::from([1, 2, 3, 4]); + let b = u16x4::from([0, 3, 2, 5]); + let e = u16x4::from([1, 3, 0, 2]); + let r: u16x4 = transmute(vpmin_u16(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmin_u32() { + let a = u32x2::from([1, 2]); + let b = u32x2::from([0, 3]); + let e = u32x2::from([1, 0]); + let r: u32x2 = transmute(vpmin_u32(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmin_f32() { + let a = f32x2::from([1., -2.]); + let b = f32x2::from([0., 3.]); + let e = f32x2::from([-2., 0.]); + let r: f32x2 = transmute(vpmin_f32(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmax_s8() { + let a = i8x8::from([1, -2, 3, -4, 5, 6, 7, 8]); + let b = i8x8::from([0, 3, 2, 5, 4, 7, 6, 9]); + let e = i8x8::from([1, 3, 6, 8, 3, 5, 7, 9]); + let r: i8x8 = transmute(vpmax_s8(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmax_s16() { + let a = i16x4::from([1, 2, 3, -4]); + let b = i16x4::from([0, 3, 2, 5]); + let e = i16x4::from([2, 3, 3, 5]); + let r: i16x4 = transmute(vpmax_s16(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmax_s32() { + let a = i32x2::from([1, -2]); + let b = i32x2::from([0, 3]); + let e = i32x2::from([1, 3]); + let r: i32x2 = transmute(vpmax_s32(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmax_u8() { + let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 8]); + let b = u8x8::from([0, 3, 2, 5, 4, 7, 6, 9]); + let e = u8x8::from([2, 4, 6, 8, 3, 5, 7, 9]); + let r: u8x8 = transmute(vpmax_u8(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmax_u16() { + let a = u16x4::from([1, 2, 3, 4]); + let b = u16x4::from([0, 3, 2, 5]); + let e = u16x4::from([2, 4, 3, 5]); + let r: u16x4 = transmute(vpmax_u16(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmax_u32() { + let a = u32x2::from([1, 2]); + let b = u32x2::from([0, 3]); + let e = u32x2::from([2, 3]); + let r: u32x2 = transmute(vpmax_u32(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpmax_f32() { + let a = f32x2::from([1., -2.]); + let b = f32x2::from([0., 3.]); + let e = f32x2::from([1., 3.]); + let r: f32x2 = transmute(vpmax_f32(transmute(a), transmute(b))); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpadd_s16() { + let a = i16x4::from([1, 2, 3, 4]); + let b = i16x4::from([0, -1, -2, -3]); + let r: i16x4 = transmute(vpadd_s16(transmute(a), transmute(b))); + let e = i16x4::from([3, 7, -1, -5]); + assert_eq!(r, e); +} +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpadd_s32() { + let a = i32x2::from([1, 2]); + let b = i32x2::from([0, -1]); + let r: i32x2 = transmute(vpadd_s32(transmute(a), transmute(b))); + let e = i32x2::from([3, -1]); + assert_eq!(r, e); +} +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpadd_s8() { + let a = i8x8::from([1, 2, 3, 4, 5, 6, 7, 8]); + let b = i8x8::from([0, -1, -2, -3, -4, -5, -6, -7]); + let r: i8x8 = transmute(vpadd_s8(transmute(a), transmute(b))); + let e = i8x8::from([3, 7, 11, 15, -1, -5, -9, -13]); + assert_eq!(r, e); +} +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpadd_u16() { + let a = u16x4::from([1, 2, 3, 4]); + let b = u16x4::from([30, 31, 32, 33]); + let r: u16x4 = transmute(vpadd_u16(transmute(a), transmute(b))); + let e = u16x4::from([3, 7, 61, 65]); + assert_eq!(r, e); +} +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpadd_u32() { + let a = u32x2::from([1, 2]); + let b = u32x2::from([30, 31]); + let r: u32x2 = transmute(vpadd_u32(transmute(a), transmute(b))); + let e = u32x2::from([3, 61]); + assert_eq!(r, e); +} +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpadd_u8() { + let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 8]); + let b = u8x8::from([30, 31, 32, 33, 34, 35, 36, 37]); + let r: u8x8 = transmute(vpadd_u8(transmute(a), transmute(b))); + let e = u8x8::from([3, 7, 11, 15, 61, 65, 69, 73]); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqsub_u8() { + let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 0xff]); + let b = u8x8::from([30, 1, 1, 1, 34, 0xff, 36, 37]); + let r: u8x8 = transmute(vqsub_u8(transmute(a), transmute(b))); + let e = u8x8::from([0, 1, 2, 3, 0, 0, 0, 218]); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqadd_u8() { + let a = u8x8::from([1, 2, 3, 4, 5, 6, 7, 0xff]); + let b = u8x8::from([30, 1, 1, 1, 34, 0xff, 36, 37]); + let r: u8x8 = transmute(vqadd_u8(transmute(a), transmute(b))); + let e = u8x8::from([31, 3, 4, 5, 39, 0xff, 43, 0xff]); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +fn main() { + unsafe { + test_vpmin_s8(); + test_vpmin_s16(); + test_vpmin_s32(); + test_vpmin_u8(); + test_vpmin_u16(); + test_vpmin_u32(); + test_vpmin_f32(); + test_vpmax_s8(); + test_vpmax_s16(); + test_vpmax_s32(); + test_vpmax_u8(); + test_vpmax_u16(); + test_vpmax_u32(); + test_vpmax_f32(); + + test_vpadd_s16(); + test_vpadd_s32(); + test_vpadd_s8(); + test_vpadd_u16(); + test_vpadd_u32(); + test_vpadd_u8(); + + test_vqsub_u8(); + test_vqadd_u8(); + } +} + +#[cfg(not(target_arch = "aarch64"))] +fn main() {} diff --git a/compiler/rustc_codegen_cranelift/example/polymorphize_coroutine.rs b/compiler/rustc_codegen_cranelift/example/polymorphize_coroutine.rs new file mode 100644 index 00000000000..407da94c0f0 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/polymorphize_coroutine.rs @@ -0,0 +1,17 @@ +#![feature(coroutines, coroutine_trait, stmt_expr_attributes)] + +use std::ops::Coroutine; +use std::pin::Pin; + +fn main() { + run_coroutine::<i32>(); +} + +fn run_coroutine<T>() { + let mut coroutine = #[coroutine] + || { + yield; + return; + }; + Pin::new(&mut coroutine).resume(()); +} diff --git a/compiler/rustc_codegen_cranelift/example/raw-dylib.rs b/compiler/rustc_codegen_cranelift/example/raw-dylib.rs new file mode 100644 index 00000000000..4711884f76a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/raw-dylib.rs @@ -0,0 +1,31 @@ +// Tests the raw-dylib feature for Windows. +// https://doc.rust-lang.org/reference/items/external-blocks.html#the-link-attribute + +fn main() { + #[cfg(windows)] + { + #[link(name = "kernel32", kind = "raw-dylib")] + extern "C" { + fn GetModuleFileNameA( + module: *mut std::ffi::c_void, + filename: *mut u8, + size: u32, + ) -> u32; + } + + // Get the filename of the current executable.... + let mut buffer = [0u8; 1024]; + let size = unsafe { + GetModuleFileNameA(core::ptr::null_mut(), buffer.as_mut_ptr(), buffer.len() as u32) + }; + if size == 0 { + eprintln!("failed to get module file name: {}", std::io::Error::last_os_error()); + return; + } else { + // ...and make sure that it matches the test name. + let filename = + std::ffi::CStr::from_bytes_with_nul(&buffer[..size as usize + 1]).unwrap(); + assert!(filename.to_str().unwrap().ends_with("raw-dylib.exe")); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs new file mode 100644 index 00000000000..e99763e2722 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/std_example.rs @@ -0,0 +1,553 @@ +#![feature( + core_intrinsics, + coroutines, + stmt_expr_attributes, + coroutine_trait, + repr_simd, + tuple_trait, + unboxed_closures +)] +#![allow(internal_features)] + +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; +use std::hint::black_box; +use std::io::Write; +use std::ops::Coroutine; + +fn main() { + println!("{:?}", std::env::args().collect::<Vec<_>>()); + + let mutex = std::sync::Mutex::new(()); + let _guard = mutex.lock().unwrap(); + + let _ = ::std::iter::repeat('a' as u8).take(10).collect::<Vec<_>>(); + let stderr = ::std::io::stderr(); + let mut stderr = stderr.lock(); + + std::thread::spawn(move || { + println!("Hello from another thread!"); + }); + + writeln!(stderr, "some {} text", "<unknown>").unwrap(); + + let _ = std::process::Command::new("true").env("c", "d").spawn(); + + println!("cargo:rustc-link-lib=z"); + + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| {}); + + let _eq = LoopState::Continue(()) == LoopState::Break(()); + + // Make sure ByValPair values with differently sized components are correctly passed + map(None::<(u8, Box<Instruction>)>); + + println!("{}", 2.3f32.exp()); + println!("{}", 2.3f32.exp2()); + println!("{}", 2.3f32.abs()); + println!("{}", 2.3f32.sqrt()); + println!("{}", 2.3f32.floor()); + println!("{}", 2.3f32.ceil()); + println!("{}", 2.3f32.min(1.0)); + println!("{}", 2.3f32.max(1.0)); + println!("{}", 2.3f32.powi(2)); + println!("{}", 2.3f32.log2()); + assert_eq!(2.3f32.copysign(-1.0), -2.3f32); + println!("{}", 2.3f32.powf(2.0)); + + assert_eq!(i64::MAX.checked_mul(2), None); + + assert_eq!(-128i8, (-128i8).saturating_sub(1)); + assert_eq!(127i8, 127i8.saturating_sub(-128)); + assert_eq!(-128i8, (-128i8).saturating_add(-128)); + assert_eq!(127i8, 127i8.saturating_add(1)); + + assert_eq!(0b0000000000000000000000000010000010000000000000000000000000000000_0000000000100000000000000000000000001000000000000100000000000000u128.leading_zeros(), 26); + assert_eq!(0b0000000000000000000000000010000000000000000000000000000000000000_0000000000000000000000000000000000001000000000000000000010000000u128.trailing_zeros(), 7); + assert_eq!( + core::intrinsics::saturating_sub(0, -170141183460469231731687303715884105728i128), + 170141183460469231731687303715884105727i128 + ); + + std::hint::black_box(std::hint::black_box(7571400400375753350092698930310845914i128) * 10); + assert!(0i128.checked_div(2i128).is_some()); + assert!(0u128.checked_div(2u128).is_some()); + assert_eq!(1u128 + 2, 3); + + assert_eq!(0b100010000000000000000000000000000u128 >> 10, 0b10001000000000000000000u128); + assert_eq!(0xFEDCBA987654321123456789ABCDEFu128 >> 64, 0xFEDCBA98765432u128); + assert_eq!(0xFEDCBA987654321123456789ABCDEFu128 as i128 >> 64, 0xFEDCBA98765432i128); + + let tmp = 353985398u128; + assert_eq!(tmp * 932490u128, 330087843781020u128); + + let tmp = -0x1234_5678_9ABC_DEF0i64; + assert_eq!(tmp as i128, -0x1234_5678_9ABC_DEF0i128); + + // Check that all u/i128 <-> float casts work correctly. + let houndred_u128 = 100u128; + let houndred_i128 = 100i128; + let houndred_f32 = 100.0f32; + let houndred_f64 = 100.0f64; + assert_eq!(houndred_u128 as f32, 100.0); + assert_eq!(houndred_u128 as f64, 100.0); + assert_eq!(houndred_f32 as u128, 100); + assert_eq!(houndred_f64 as u128, 100); + assert_eq!(houndred_i128 as f32, 100.0); + assert_eq!(houndred_i128 as f64, 100.0); + assert_eq!(houndred_f32 as i128, 100); + assert_eq!(houndred_f64 as i128, 100); + assert_eq!(1u128.rotate_left(2), 4); + + assert_eq!(black_box(f32::NAN) as i128, 0); + assert_eq!(black_box(f32::NAN) as u128, 0); + + // Test signed 128bit comparing + let max = usize::MAX as i128; + if 100i128 < 0i128 || 100i128 > max { + panic!(); + } + + test_checked_mul(); + + let _a = 1u32 << 2u8; + + let empty: [i32; 0] = []; + assert!(empty.is_sorted()); + + println!("{:?}", std::intrinsics::caller_location()); + + #[cfg(target_arch = "x86_64")] + unsafe { + test_simd(); + } + + Box::pin( + #[coroutine] + move |mut _task_context| { + yield (); + }, + ) + .as_mut() + .resume(0); + + #[derive(Copy, Clone)] + enum Nums { + NegOne = -1, + } + + let kind = Nums::NegOne; + assert_eq!(-1i128, kind as i128); + + let options = [1u128]; + match options[0] { + 1 => (), + 0 => loop {}, + v => panic(v), + }; + + if black_box(false) { + // Based on https://github.com/rust-lang/rust/blob/2f320a224e827b400be25966755a621779f797cc/src/test/ui/debuginfo/debuginfo_with_uninhabitable_field_and_unsized.rs + let _ = Foo::<dyn Send>::new(); + + #[allow(dead_code)] + struct Foo<T: ?Sized> { + base: Never, + value: T, + } + + impl<T: ?Sized> Foo<T> { + pub fn new() -> Box<Foo<T>> { + todo!() + } + } + + enum Never {} + } + + foo(I64X2(0, 0)); + + transmute_fat_pointer(); + + rust_call_abi(); + + const fn no_str() -> Option<Box<str>> { + None + } + + static STATIC_WITH_MAYBE_NESTED_BOX: &Option<Box<str>> = &no_str(); + + println!("{:?}", STATIC_WITH_MAYBE_NESTED_BOX); +} + +fn panic(_: u128) { + panic!(); +} + +use std::mem::transmute; + +#[cfg(target_pointer_width = "32")] +type TwoPtrs = i64; +#[cfg(target_pointer_width = "64")] +type TwoPtrs = i128; + +fn transmute_fat_pointer() -> TwoPtrs { + unsafe { transmute::<_, TwoPtrs>("true !") } +} + +extern "rust-call" fn rust_call_abi_callee<T: std::marker::Tuple>(_: T) {} + +fn rust_call_abi() { + rust_call_abi_callee(()); + rust_call_abi_callee((1, 2)); +} + +#[repr(simd)] +struct I64X2(i64, i64); + +#[allow(improper_ctypes_definitions)] +extern "C" fn foo(_a: I64X2) {} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse4.2")] +#[cfg(not(jit))] +unsafe fn test_crc32() { + assert!(is_x86_feature_detected!("sse4.2")); + + let a = 42u32; + let b = 0xdeadbeefu64; + + assert_eq!(_mm_crc32_u8(a, b as u8), 4135334616); + assert_eq!(_mm_crc32_u16(a, b as u16), 1200687288); + assert_eq!(_mm_crc32_u32(a, b as u32), 2543798776); + assert_eq!(_mm_crc32_u64(a as u64, b as u64), 241952147); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_simd() { + assert!(is_x86_feature_detected!("sse2")); + + let x = _mm_setzero_si128(); + let y = _mm_set1_epi16(7); + let or = _mm_or_si128(x, y); + let cmp_eq = _mm_cmpeq_epi8(y, y); + let cmp_lt = _mm_cmplt_epi8(y, y); + + let (zero0, zero1) = std::mem::transmute::<_, (u64, u64)>(x); + assert_eq!((zero0, zero1), (0, 0)); + assert_eq!(std::mem::transmute::<_, [u16; 8]>(or), [7, 7, 7, 7, 7, 7, 7, 7]); + assert_eq!( + std::mem::transmute::<_, [u16; 8]>(cmp_eq), + [0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff] + ); + assert_eq!(std::mem::transmute::<_, [u16; 8]>(cmp_lt), [0, 0, 0, 0, 0, 0, 0, 0]); + + test_mm_slli_si128(); + test_mm_movemask_epi8(); + test_mm256_movemask_epi8(); + test_mm_add_epi8(); + test_mm_add_pd(); + test_mm_cvtepi8_epi16(); + #[cfg(not(jit))] + test_mm_cvtps_epi32(); + test_mm_cvttps_epi32(); + test_mm_cvtsi128_si64(); + + test_mm_extract_epi8(); + test_mm_insert_epi16(); + test_mm_shuffle_epi8(); + + test_mm256_shuffle_epi8(); + test_mm256_permute2x128_si256(); + test_mm256_permutevar8x32_epi32(); + + #[rustfmt::skip] + let mask1 = _mm_movemask_epi8(dbg!(_mm_setr_epi8(255u8 as i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))); + assert_eq!(mask1, 1); + + #[cfg(not(jit))] + test_crc32(); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_slli_si128() { + #[rustfmt::skip] + let a = _mm_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ); + let r = _mm_slli_si128(a, 1); + let e = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + + #[rustfmt::skip] + let a = _mm_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ); + let r = _mm_slli_si128(a, 15); + let e = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); + assert_eq_m128i(r, e); + + #[rustfmt::skip] + let a = _mm_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ); + let r = _mm_slli_si128(a, 16); + assert_eq_m128i(r, _mm_set1_epi8(0)); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_movemask_epi8() { + #[rustfmt::skip] + let a = _mm_setr_epi8( + 0b1000_0000u8 as i8, 0b0, 0b1000_0000u8 as i8, 0b01, + 0b0101, 0b1111_0000u8 as i8, 0, 0, + 0, 0, 0b1111_0000u8 as i8, 0b0101, + 0b01, 0b1000_0000u8 as i8, 0b0, 0b1000_0000u8 as i8, + ); + let r = _mm_movemask_epi8(a); + assert_eq!(r, 0b10100100_00100101); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn test_mm256_movemask_epi8() { + let a = _mm256_set1_epi8(-1); + let r = _mm256_movemask_epi8(a); + let e = -1; + assert_eq!(r, e); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_add_epi8() { + let a = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm_setr_epi8( + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + let r = _mm_add_epi8(a, b); + #[rustfmt::skip] + let e = _mm_setr_epi8( + 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, + ); + assert_eq_m128i(r, e); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_add_pd() { + let a = _mm_setr_pd(1.0, 2.0); + let b = _mm_setr_pd(5.0, 10.0); + let r = _mm_add_pd(a, b); + assert_eq_m128d(r, _mm_setr_pd(6.0, 12.0)); +} + +#[cfg(target_arch = "x86_64")] +fn assert_eq_m128i(x: std::arch::x86_64::__m128i, y: std::arch::x86_64::__m128i) { + unsafe { + assert_eq!(std::mem::transmute::<_, [u8; 16]>(x), std::mem::transmute::<_, [u8; 16]>(y)); + } +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +pub unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { + if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { + panic!("{:?} != {:?}", a, b); + } +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx")] +pub unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) { + assert_eq!(std::mem::transmute::<_, [u64; 4]>(a), std::mem::transmute::<_, [u64; 4]>(b)) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_cvtsi128_si64() { + let r = _mm_cvtsi128_si64(std::mem::transmute::<[i64; 2], _>([5, 0])); + assert_eq!(r, 5); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse4.1")] +unsafe fn test_mm_cvtepi8_epi16() { + let a = _mm_set1_epi8(10); + let r = _mm_cvtepi8_epi16(a); + let e = _mm_set1_epi16(10); + assert_eq_m128i(r, e); + let a = _mm_set1_epi8(-10); + let r = _mm_cvtepi8_epi16(a); + let e = _mm_set1_epi16(-10); + assert_eq_m128i(r, e); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse4.1")] +unsafe fn test_mm_extract_epi8() { + #[rustfmt::skip] + let a = _mm_setr_epi8( + -1, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 + ); + let r1 = _mm_extract_epi8(a, 0); + let r2 = _mm_extract_epi8(a, 3); + assert_eq!(r1, 0xFF); + assert_eq!(r2, 3); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_insert_epi16() { + let a = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_insert_epi16(a, 9, 0); + let e = _mm_setr_epi16(9, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "ssse3")] +unsafe fn test_mm_shuffle_epi8() { + #[rustfmt::skip] + let a = _mm_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, + ); + #[rustfmt::skip] + let b = _mm_setr_epi8( + 4, 128_u8 as i8, 4, 3, + 24, 12, 6, 19, + 12, 5, 5, 10, + 4, 1, 8, 0, + ); + let expected = _mm_setr_epi8(5, 0, 5, 4, 9, 13, 7, 4, 13, 6, 6, 11, 5, 2, 9, 1); + let r = _mm_shuffle_epi8(a, b); + assert_eq_m128i(r, expected); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn test_mm256_shuffle_epi8() { + #[rustfmt::skip] + let a = _mm256_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, + ); + #[rustfmt::skip] + let b = _mm256_setr_epi8( + 4, 128u8 as i8, 4, 3, 24, 12, 6, 19, + 12, 5, 5, 10, 4, 1, 8, 0, + 4, 128u8 as i8, 4, 3, 24, 12, 6, 19, + 12, 5, 5, 10, 4, 1, 8, 0, + ); + #[rustfmt::skip] + let expected = _mm256_setr_epi8( + 5, 0, 5, 4, 9, 13, 7, 4, + 13, 6, 6, 11, 5, 2, 9, 1, + 21, 0, 21, 20, 25, 29, 23, 20, + 29, 22, 22, 27, 21, 18, 25, 17, + ); + let r = _mm256_shuffle_epi8(a, b); + assert_eq_m256i(r, expected); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn test_mm256_permute2x128_si256() { + let a = _mm256_setr_epi64x(100, 200, 500, 600); + let b = _mm256_setr_epi64x(300, 400, 700, 800); + let r = _mm256_permute2x128_si256::<0b00_01_00_11>(a, b); + let e = _mm256_setr_epi64x(700, 800, 500, 600); + assert_eq_m256i(r, e); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn test_mm256_permutevar8x32_epi32() { + let a = _mm256_setr_epi32(100, 200, 300, 400, 500, 600, 700, 800); + let idx = _mm256_setr_epi32(7, 6, 5, 4, 3, 2, 1, 0); + let r = _mm256_setr_epi32(800, 700, 600, 500, 400, 300, 200, 100); + let e = _mm256_permutevar8x32_epi32(a, idx); + assert_eq_m256i(r, e); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +#[cfg(not(jit))] +unsafe fn test_mm_cvtps_epi32() { + let floats: [f32; 4] = [1.5, -2.5, i32::MAX as f32 + 1.0, f32::NAN]; + + let float_vec = _mm_loadu_ps(floats.as_ptr()); + let int_vec = _mm_cvtps_epi32(float_vec); + + let mut ints: [i32; 4] = [0; 4]; + _mm_storeu_si128(ints.as_mut_ptr() as *mut __m128i, int_vec); + + // this is very different from `floats.map(|f| f as i32)`! + let expected_ints: [i32; 4] = [2, -2, i32::MIN, i32::MIN]; + + assert_eq!(ints, expected_ints); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn test_mm_cvttps_epi32() { + let floats: [f32; 4] = [1.5, -2.5, i32::MAX as f32 + 1.0, f32::NAN]; + + let float_vec = _mm_loadu_ps(floats.as_ptr()); + let int_vec = _mm_cvttps_epi32(float_vec); + + let mut ints: [i32; 4] = [0; 4]; + _mm_storeu_si128(ints.as_mut_ptr() as *mut __m128i, int_vec); + + // this is very different from `floats.map(|f| f as i32)`! + let expected_ints: [i32; 4] = [1, -2, i32::MIN, i32::MIN]; + + assert_eq!(ints, expected_ints); +} + +fn test_checked_mul() { + let u: Option<u8> = u8::from_str_radix("1000", 10).ok(); + assert_eq!(u, None); + + assert_eq!(1u8.checked_mul(255u8), Some(255u8)); + assert_eq!(255u8.checked_mul(255u8), None); + assert_eq!(1i8.checked_mul(127i8), Some(127i8)); + assert_eq!(127i8.checked_mul(127i8), None); + assert_eq!((-1i8).checked_mul(-127i8), Some(127i8)); + assert_eq!(1i8.checked_mul(-128i8), Some(-128i8)); + assert_eq!((-128i8).checked_mul(-128i8), None); + + assert_eq!(1u64.checked_mul(u64::MAX), Some(u64::MAX)); + assert_eq!(u64::MAX.checked_mul(u64::MAX), None); + assert_eq!(1i64.checked_mul(i64::MAX), Some(i64::MAX)); + assert_eq!(i64::MAX.checked_mul(i64::MAX), None); + assert_eq!((-1i64).checked_mul(i64::MIN + 1), Some(i64::MAX)); + assert_eq!(1i64.checked_mul(i64::MIN), Some(i64::MIN)); + assert_eq!(i64::MIN.checked_mul(i64::MIN), None); +} + +#[derive(PartialEq)] +enum LoopState { + Continue(()), + Break(()), +} + +pub enum Instruction { + Increment, + Loop, +} + +fn map(a: Option<(u8, Box<Instruction>)>) -> Option<Box<Instruction>> { + match a { + None => None, + Some((_, instr)) => Some(instr), + } +} diff --git a/compiler/rustc_codegen_cranelift/example/subslice-patterns-const-eval.rs b/compiler/rustc_codegen_cranelift/example/subslice-patterns-const-eval.rs new file mode 100644 index 00000000000..3c878916663 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/subslice-patterns-const-eval.rs @@ -0,0 +1,102 @@ +// Based on https://github.com/rust-lang/rust/blob/c5840f9d252c2f5cc16698dbf385a29c5de3ca07/src/test/ui/array-slice-vec/subslice-patterns-const-eval-match.rs + +// Test that array subslice patterns are correctly handled in const evaluation. + +// run-pass + +#[derive(PartialEq, Debug, Clone)] +struct N(u8); + +#[derive(PartialEq, Debug, Clone)] +struct Z; + +macro_rules! n { + ($($e:expr),* $(,)?) => { + [$(N($e)),*] + } +} + +// This macro has an unused variable so that it can be repeated base on the +// number of times a repeated variable (`$e` in `z`) occurs. +macro_rules! zed { + ($e:expr) => { + Z + }; +} + +macro_rules! z { + ($($e:expr),* $(,)?) => { + [$(zed!($e)),*] + } +} + +// Compare constant evaluation and runtime evaluation of a given expression. +macro_rules! compare_evaluation { + ($e:expr, $t:ty $(,)?) => {{ + const CONST_EVAL: $t = $e; + const fn const_eval() -> $t { + $e + } + static CONST_EVAL2: $t = const_eval(); + let runtime_eval = $e; + assert_eq!(CONST_EVAL, runtime_eval); + assert_eq!(CONST_EVAL2, runtime_eval); + }}; +} + +// Repeat `$test`, substituting the given macro variables with the given +// identifiers. +// +// For example: +// +// repeat! { +// ($name); X; Y: +// struct $name; +// } +// +// Expands to: +// +// struct X; struct Y; +// +// This is used to repeat the tests using both the `N` and `Z` +// types. +macro_rules! repeat { + (($($dollar:tt $placeholder:ident)*); $($($values:ident),+);*: $($test:tt)*) => { + macro_rules! single { + ($($dollar $placeholder:ident),*) => { $($test)* } + } + $(single!($($values),+);)* + } +} + +#[rustfmt::skip] +fn main() { + repeat! { + ($arr $Ty); n, N; z, Z: + compare_evaluation!({ let [_, x @ .., _] = $arr!(1, 2, 3, 4); x }, [$Ty; 2]); + compare_evaluation!({ let [_, ref x @ .., _] = $arr!(1, 2, 3, 4); x }, &'static [$Ty; 2]); + compare_evaluation!({ let [_, x @ .., _] = &$arr!(1, 2, 3, 4); x }, &'static [$Ty; 2]); + + compare_evaluation!({ let [_, _, x @ .., _, _] = $arr!(1, 2, 3, 4); x }, [$Ty; 0]); + compare_evaluation!( + { let [_, _, ref x @ .., _, _] = $arr!(1, 2, 3, 4); x }, + &'static [$Ty; 0], + ); + compare_evaluation!( + { let [_, _, x @ .., _, _] = &$arr!(1, 2, 3, 4); x }, + &'static [$Ty; 0], + ); + + compare_evaluation!({ let [_, .., x] = $arr!(1, 2, 3, 4); x }, $Ty); + compare_evaluation!({ let [_, .., ref x] = $arr!(1, 2, 3, 4); x }, &'static $Ty); + compare_evaluation!({ let [_, _y @ .., x] = &$arr!(1, 2, 3, 4); x }, &'static $Ty); + } + + compare_evaluation!({ let [_, .., N(x)] = n!(1, 2, 3, 4); x }, u8); + compare_evaluation!({ let [_, .., N(ref x)] = n!(1, 2, 3, 4); x }, &'static u8); + compare_evaluation!({ let [_, .., N(x)] = &n!(1, 2, 3, 4); x }, &'static u8); + + compare_evaluation!({ let [N(x), .., _] = n!(1, 2, 3, 4); x }, u8); + compare_evaluation!({ let [N(ref x), .., _] = n!(1, 2, 3, 4); x }, &'static u8); + compare_evaluation!({ let [N(x), .., _] = &n!(1, 2, 3, 4); x }, &'static u8); +} diff --git a/compiler/rustc_codegen_cranelift/example/track-caller-attribute.rs b/compiler/rustc_codegen_cranelift/example/track-caller-attribute.rs new file mode 100644 index 00000000000..93bab17e46b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/track-caller-attribute.rs @@ -0,0 +1,40 @@ +// Based on https://github.com/anp/rust/blob/175631311716d7dfeceec40d2587cde7142ffa8c/src/test/ui/rfc-2091-track-caller/track-caller-attribute.rs + +// run-pass + +use std::panic::Location; + +#[track_caller] +fn tracked() -> &'static Location<'static> { + Location::caller() +} + +fn nested_intrinsic() -> &'static Location<'static> { + Location::caller() +} + +fn nested_tracked() -> &'static Location<'static> { + tracked() +} + +fn main() { + let location = Location::caller(); + assert_eq!(location.file(), file!()); + assert_eq!(location.line(), 21); + assert_eq!(location.column(), 20); + + let tracked = tracked(); + assert_eq!(tracked.file(), file!()); + assert_eq!(tracked.line(), 26); + assert_eq!(tracked.column(), 19); + + let nested = nested_intrinsic(); + assert_eq!(nested.file(), file!()); + assert_eq!(nested.line(), 13); + assert_eq!(nested.column(), 5); + + let contained = nested_tracked(); + assert_eq!(contained.file(), file!()); + assert_eq!(contained.line(), 17); + assert_eq!(contained.column(), 5); +} diff --git a/compiler/rustc_codegen_cranelift/patches/0002-abi-cafe-Disable-broken-tests.patch b/compiler/rustc_codegen_cranelift/patches/0002-abi-cafe-Disable-broken-tests.patch new file mode 100644 index 00000000000..8a2565f1668 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0002-abi-cafe-Disable-broken-tests.patch @@ -0,0 +1,75 @@ +From 236df390f3bc4ed69c26f4d51d584bea246da886 Mon Sep 17 00:00:00 2001 +From: bjorn3 <17426603+bjorn3@users.noreply.github.com> +Date: Tue, 9 Jul 2024 11:25:14 +0000 +Subject: [PATCH] Disable broken tests + +--- + src/report.rs | 36 ++++++++++++++++++++++++++++++++++++ + 1 file changed, 36 insertions(+) + +diff --git a/src/report.rs b/src/report.rs +index 958ab43..dcf1044 100644 +--- a/src/report.rs ++++ b/src/report.rs +@@ -48,6 +48,58 @@ pub fn get_test_rules(test: &TestKey, caller: &dyn Toolchain, callee: &dyn Toolc + // + // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES + ++ if cfg!(all(target_arch = "aarch64", target_os = "linux")) { ++ if test.test == "F32Array" && test.options.convention == CallingConvention::C { ++ result.check = Busted(Check); ++ } ++ ++ if test.test == "OptionU128" && test.options.convention == CallingConvention::Rust && test.options.repr == LangRepr::C { ++ result.check = Busted(Check); ++ } ++ } ++ ++ if cfg!(all(target_arch = "aarch64", target_os = "macos")) { ++ if test.test == "SingleVariantUnion" && test.options.convention == CallingConvention::C && test.options.repr == LangRepr::C { ++ result.check = Busted(Check); ++ } ++ ++ if test.test == "OptionU128" && test.caller == "rustc" && test.options.convention == CallingConvention::Rust && test.options.repr == LangRepr::C { ++ result.check = Busted(Run); ++ } ++ ++ if test.test == "OptionU128" && test.caller == "cgclif" && test.options.convention == CallingConvention::Rust && test.options.repr == LangRepr::C { ++ result.check = Busted(Check); ++ } ++ } ++ ++ if cfg!(all(target_arch = "x86_64", unix)) { ++ if test.test == "OptionU128" && test.options.convention == CallingConvention::Rust && test.options.repr == LangRepr::Rust { ++ result.check = Busted(Run); ++ } ++ } ++ ++ if cfg!(all(target_arch = "x86_64", windows)) { ++ if test.test == "OptionU128" && test.options.convention == CallingConvention::Rust { ++ result.check = Busted(Check); ++ } ++ ++ if test.test == "OptionU128" && test.options.convention == CallingConvention::Rust && (test.caller == "rustc" || test.options.repr == LangRepr::Rust) { ++ result.check = Busted(Run); ++ } ++ ++ if test.test == "simple" && test.options.convention == CallingConvention::Rust { ++ result.check = Busted(Check); ++ } ++ ++ if test.test == "simple" && test.options.convention == CallingConvention::Rust && test.caller == "rustc" { ++ result.check = Busted(Run); ++ } ++ } ++ ++ if test.test == "f16" || test.test == "f128" { ++ result.run = Skip; ++ } ++ + // END OF VENDOR RESERVED AREA + // + // +-- +2.34.1 + diff --git a/compiler/rustc_codegen_cranelift/patches/0022-coretests-Disable-not-compiling-tests.patch b/compiler/rustc_codegen_cranelift/patches/0022-coretests-Disable-not-compiling-tests.patch new file mode 100644 index 00000000000..8c404956bcc --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0022-coretests-Disable-not-compiling-tests.patch @@ -0,0 +1,44 @@ +From f6befc4bb51d84f5f1cf35938a168c953d421350 Mon Sep 17 00:00:00 2001 +From: bjorn3 <bjorn3@users.noreply.github.com> +Date: Sun, 24 Nov 2019 15:10:23 +0100 +Subject: [PATCH] [core] Disable not compiling tests + +--- + library/core/tests/Cargo.toml | 8 ++++++++ + library/core/tests/num/flt2dec/mod.rs | 1 - + library/core/tests/num/int_macros.rs | 2 ++ + library/core/tests/num/uint_macros.rs | 2 ++ + library/core/tests/ptr.rs | 2 ++ + library/core/tests/slice.rs | 2 ++ + 6 files changed, 16 insertions(+), 1 deletion(-) + create mode 100644 library/core/tests/Cargo.toml + +diff --git a/Cargo.toml b/Cargo.toml +new file mode 100644 +index 0000000..46fd999 +--- /dev/null ++++ b/Cargo.toml +@@ -0,0 +1,12 @@ ++[package] ++name = "coretests" ++version = "0.0.0" ++edition = "2021" ++ ++[lib] ++name = "coretests" ++path = "lib.rs" ++ ++[dependencies] ++rand = { version = "0.8.5", default-features = false } ++rand_xorshift = { version = "0.3.0", default-features = false } +diff --git a/lib.rs b/lib.rs +index 42a26ae..5ac1042 100644 +--- a/lib.rs ++++ b/lib.rs +@@ -1,3 +1,4 @@ ++#![cfg(test)] + // tidy-alphabetical-start + #![cfg_attr(bootstrap, feature(offset_of_nested))] + #![cfg_attr(target_has_atomic = "128", feature(integer_atomics))] +-- +2.21.0 (Apple Git-122) diff --git a/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch new file mode 100644 index 00000000000..d579c9588f0 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch @@ -0,0 +1,42 @@ +From ad7ffe71baba46865f2e65266ab025920dfdc20b Mon Sep 17 00:00:00 2001 +From: bjorn3 <bjorn3@users.noreply.github.com> +Date: Thu, 18 Feb 2021 18:45:28 +0100 +Subject: [PATCH] Disable 128bit atomic operations + +Cranelift doesn't support them yet +--- + library/core/src/panic/unwind_safe.rs | 6 ----- + library/core/src/sync/atomic.rs | 38 --------------------------- + library/core/tests/atomic.rs | 4 --- + 4 files changed, 4 insertions(+), 50 deletions(-) + +diff --git a/lib.rs b/lib.rs +index 1e336bf..35e6f54 100644 +--- a/lib.rs ++++ b/lib.rs +@@ -1,7 +1,6 @@ + #![cfg(test)] + // tidy-alphabetical-start + #![cfg_attr(bootstrap, feature(offset_of_nested))] +-#![cfg_attr(target_has_atomic = "128", feature(integer_atomics))] + #![cfg_attr(test, feature(cfg_match))] + #![feature(alloc_layout_extra)] + #![feature(array_chunks)] +diff --git a/atomic.rs b/atomic.rs +index b735957..ea728b6 100644 +--- a/atomic.rs ++++ b/atomic.rs +@@ -185,10 +185,6 @@ fn atomic_alignment() { + assert_eq!(align_of::<AtomicU64>(), size_of::<AtomicU64>()); + #[cfg(target_has_atomic = "64")] + assert_eq!(align_of::<AtomicI64>(), size_of::<AtomicI64>()); +- #[cfg(target_has_atomic = "128")] +- assert_eq!(align_of::<AtomicU128>(), size_of::<AtomicU128>()); +- #[cfg(target_has_atomic = "128")] +- assert_eq!(align_of::<AtomicI128>(), size_of::<AtomicI128>()); + #[cfg(target_has_atomic = "ptr")] + assert_eq!(align_of::<AtomicUsize>(), size_of::<AtomicUsize>()); + #[cfg(target_has_atomic = "ptr")] +-- +2.26.2.7.g19db9cfb68 + diff --git a/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch new file mode 100644 index 00000000000..a3f370af916 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch @@ -0,0 +1,100 @@ +From ad7ffe71baba46865f2e65266ab025920dfdc20b Mon Sep 17 00:00:00 2001 +From: bjorn3 <bjorn3@users.noreply.github.com> +Date: Thu, 18 Feb 2021 18:45:28 +0100 +Subject: [PATCH] Disable 128bit atomic operations + +Cranelift doesn't support them yet +--- + library/core/src/panic/unwind_safe.rs | 6 ----- + library/core/src/sync/atomic.rs | 38 --------------------------- + library/core/tests/atomic.rs | 4 --- + 4 files changed, 4 insertions(+), 50 deletions(-) + +diff --git a/library/core/src/panic/unwind_safe.rs b/library/core/src/panic/unwind_safe.rs +index 092b7cf..158cf71 100644 +--- a/library/core/src/panic/unwind_safe.rs ++++ b/library/core/src/panic/unwind_safe.rs +@@ -216,9 +216,6 @@ impl RefUnwindSafe for crate::sync::atomic::AtomicI32 {} + #[cfg(target_has_atomic_load_store = "64")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + impl RefUnwindSafe for crate::sync::atomic::AtomicI64 {} +-#[cfg(target_has_atomic_load_store = "128")] +-#[unstable(feature = "integer_atomics", issue = "99069")] +-impl RefUnwindSafe for crate::sync::atomic::AtomicI128 {} + + #[cfg(target_has_atomic_load_store = "ptr")] + #[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] +@@ -235,9 +232,6 @@ impl RefUnwindSafe for crate::sync::atomic::AtomicU32 {} + #[cfg(target_has_atomic_load_store = "64")] + #[stable(feature = "integer_atomics_stable", since = "1.34.0")] + impl RefUnwindSafe for crate::sync::atomic::AtomicU64 {} +-#[cfg(target_has_atomic_load_store = "128")] +-#[unstable(feature = "integer_atomics", issue = "99069")] +-impl RefUnwindSafe for crate::sync::atomic::AtomicU128 {} + + #[cfg(target_has_atomic_load_store = "8")] + #[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] +diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs +index d9de37e..8293fce 100644 +--- a/library/core/src/sync/atomic.rs ++++ b/library/core/src/sync/atomic.rs +@@ -2996,42 +2996,6 @@ atomic_int! { + 8, + u64 AtomicU64 + } +-#[cfg(target_has_atomic_load_store = "128")] +-atomic_int! { +- cfg(target_has_atomic = "128"), +- cfg(target_has_atomic_equal_alignment = "128"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), +- cfg_attr(not(test), rustc_diagnostic_item = "AtomicI128"), +- "i128", +- "#![feature(integer_atomics)]\n\n", +- atomic_min, atomic_max, +- 16, +- i128 AtomicI128 +-} +-#[cfg(target_has_atomic_load_store = "128")] +-atomic_int! { +- cfg(target_has_atomic = "128"), +- cfg(target_has_atomic_equal_alignment = "128"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- unstable(feature = "integer_atomics", issue = "99069"), +- rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), +- cfg_attr(not(test), rustc_diagnostic_item = "AtomicU128"), +- "u128", +- "#![feature(integer_atomics)]\n\n", +- atomic_umin, atomic_umax, +- 16, +- u128 AtomicU128 +-} + + #[cfg(target_has_atomic_load_store = "ptr")] + macro_rules! atomic_int_ptr_sized { + ( $($target_pointer_width:literal $align:literal)* ) => { $( +diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs +index 58b9ba4..91bbd0a 100644 +--- a/library/core/src/cell.rs ++++ b/library/core/src/cell.rs +@@ -2246,8 +2246,6 @@ unsafe_cell_primitive_into_inner! { + u32 "32" + i64 "64" + u64 "64" +- i128 "128" +- u128 "128" + isize "ptr" + usize "ptr" + } +-- +2.26.2.7.g19db9cfb68 + diff --git a/compiler/rustc_codegen_cranelift/patches/0028-coretests-Disable-long-running-tests.patch b/compiler/rustc_codegen_cranelift/patches/0028-coretests-Disable-long-running-tests.patch new file mode 100644 index 00000000000..440177018f4 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0028-coretests-Disable-long-running-tests.patch @@ -0,0 +1,48 @@ +From eb703e627e7a84f1cd8d0d87f0f69da1f0acf765 Mon Sep 17 00:00:00 2001 +From: bjorn3 <bjorn3@users.noreply.github.com> +Date: Fri, 3 Dec 2021 12:16:30 +0100 +Subject: [PATCH] Disable long running tests + +--- + library/core/tests/slice.rs | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/slice.rs b/slice.rs +index 8402833..84592e0 100644 +--- a/slice.rs ++++ b/slice.rs +@@ -1809,6 +1809,7 @@ fn sort_unstable() { + assert!(v == [0xDEADBEEF]); + } + ++/* + #[test] + #[cfg(not(target_arch = "wasm32"))] + #[cfg_attr(miri, ignore)] // Miri is too slow +@@ -1914,6 +1915,7 @@ fn select_nth_unstable() { + v.select_nth_unstable(0); + assert!(v == [0xDEADBEEF]); + } ++*/ + + #[test] + #[should_panic(expected = "index 0 greater than length of slice")] +@@ -2462,6 +2462,7 @@ take_tests! { + #[cfg(not(miri))] // unused in Miri + const EMPTY_MAX: &'static [()] = &[(); usize::MAX]; + ++/* + // can't be a constant due to const mutability rules + #[cfg(not(miri))] // unused in Miri + macro_rules! empty_max_mut { +@@ -2485,6 +2486,7 @@ take_tests! { + (take_mut_oob_max_range_to_inclusive, (..=usize::MAX), None, empty_max_mut!()), + (take_mut_in_bounds_max_range_from, (usize::MAX..), Some(&mut [] as _), empty_max_mut!()), + } ++*/ + + #[test] + fn test_slice_from_ptr_range() { +-- +2.26.2.7.g19db9cfb68 + diff --git a/compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch b/compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch new file mode 100644 index 00000000000..ada35145e29 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch @@ -0,0 +1,25 @@ +From 175d52c5e1779764b66777db1e6f172c2dc365ff Mon Sep 17 00:00:00 2001 +From: bjorn3 <17426603+bjorn3@users.noreply.github.com> +Date: Fri, 9 Aug 2024 15:44:51 +0000 +Subject: [PATCH] Disable f16 and f128 in compiler-builtins + +--- + library/sysroot/Cargo.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/library/sysroot/Cargo.toml b/library/sysroot/Cargo.toml +index 7165c3e48af..968552ad435 100644 +--- a/library/sysroot/Cargo.toml ++++ b/library/sysroot/Cargo.toml +@@ -11,7 +11,7 @@ test = { path = "../test" } + + # Forward features to the `std` crate as necessary + [features] +-default = ["std_detect_file_io", "std_detect_dlsym_getauxval", "panic-unwind"] ++default = ["std_detect_file_io", "std_detect_dlsym_getauxval", "panic-unwind", "compiler-builtins-no-f16-f128"] + backtrace = ["std/backtrace"] + compiler-builtins-c = ["std/compiler-builtins-c"] + compiler-builtins-mem = ["std/compiler-builtins-mem"] +-- +2.34.1 + diff --git a/compiler/rustc_codegen_cranelift/patches/bcryptprimitives.rs b/compiler/rustc_codegen_cranelift/patches/bcryptprimitives.rs new file mode 100644 index 00000000000..4d186485aac --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/bcryptprimitives.rs @@ -0,0 +1,22 @@ +// Shim for bcryptprimitives.dll. The Wine version shipped with Ubuntu 22.04 +// doesn't support it yet. Authored by @ChrisDenton + +#![crate_type = "cdylib"] +#![allow(nonstandard_style)] + +#[no_mangle] +pub unsafe extern "system" fn ProcessPrng(mut pbData: *mut u8, mut cbData: usize) -> i32 { + while cbData > 0 { + let size = core::cmp::min(cbData, u32::MAX as usize); + RtlGenRandom(pbData, size as u32); + cbData -= size; + pbData = pbData.add(size); + } + 1 +} + +#[link(name = "advapi32")] +extern "system" { + #[link_name = "SystemFunction036"] + pub fn RtlGenRandom(RandomBuffer: *mut u8, RandomBufferLength: u32) -> u8; +} diff --git a/compiler/rustc_codegen_cranelift/patches/coretests-lock.toml b/compiler/rustc_codegen_cranelift/patches/coretests-lock.toml new file mode 100644 index 00000000000..af8f28a193b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/coretests-lock.toml @@ -0,0 +1,35 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "coretests" +version = "0.0.0" +dependencies = [ + "rand", + "rand_xorshift", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] diff --git a/compiler/rustc_codegen_cranelift/patches/rand-lock.toml b/compiler/rustc_codegen_cranelift/patches/rand-lock.toml new file mode 100644 index 00000000000..815b828a68b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/rand-lock.toml @@ -0,0 +1,848 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2135563fb5c609d2b2b87c1e8ce7bc41b0b45430fa9661f457981503dd5bf0" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "average" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843ec791d3f24503bbf72bbd5e49a3ab4dbb4bcd0a8ef6b0c908efa73caa27b1" +dependencies = [ + "easy-cast", + "float-ord", + "num-traits", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "bitflags", + "clap_lex", + "indexmap", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "easy-cast" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd102ee8c418348759919b83b81cdbdc933ffe29740b903df448b4bafaa348e" +dependencies = [ + "libm", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" + +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num-traits" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.2", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "os_str_bytes" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" + +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.9.0" +dependencies = [ + "bincode", + "criterion", + "libc", + "log", + "rand_chacha", + "rand_core", + "rand_pcg", + "rayon", + "serde", + "zerocopy", +] + +[[package]] +name = "rand_chacha" +version = "0.4.0" +dependencies = [ + "ppv-lite86", + "rand_core", + "serde", + "serde_json", +] + +[[package]] +name = "rand_core" +version = "0.7.0" +dependencies = [ + "getrandom", + "serde", + "zerocopy", +] + +[[package]] +name = "rand_distr" +version = "0.5.0" +dependencies = [ + "average", + "num-traits", + "rand", + "rand_pcg", + "serde", + "serde_with", + "special", +] + +[[package]] +name = "rand_pcg" +version = "0.4.0" +dependencies = [ + "bincode", + "rand", + "rand_core", + "serde", +] + +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "regex" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "special" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a65e074159b75dcf173a4733ab2188baac24967b5c8ec9ed87ae15fcbc7636" +dependencies = [ + "libc", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] diff --git a/compiler/rustc_codegen_cranelift/patches/regex-lock.toml b/compiler/rustc_codegen_cranelift/patches/regex-lock.toml new file mode 100644 index 00000000000..e0df6f9ae26 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/regex-lock.toml @@ -0,0 +1,457 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2135563fb5c609d2b2b87c1e8ce7bc41b0b45430fa9661f457981503dd5bf0" +dependencies = [ + "log", + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "arbitrary" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bstr" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "derive_arbitrary" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "lexopt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baff4b617f7df3d896f97fe922b64817f6cd9a756bb81d40f8883f2f66dcb401" + +[[package]] +name = "libc" +version = "0.2.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +dependencies = [ + "log", +] + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quickcheck" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +dependencies = [ + "rand", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "regex" +version = "1.9.5" +dependencies = [ + "aho-corasick", + "anyhow", + "doc-comment", + "env_logger", + "memchr", + "once_cell", + "quickcheck", + "regex-automata", + "regex-syntax", + "regex-test", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" +dependencies = [ + "aho-corasick", + "anyhow", + "bstr", + "doc-comment", + "env_logger", + "log", + "memchr", + "quickcheck", + "regex-syntax", + "regex-test", +] + +[[package]] +name = "regex-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "bstr", + "lexopt", + "log", + "memmap2", + "regex", + "regex-automata", + "regex-lite", + "regex-syntax", + "tabwriter", + "textwrap", +] + +[[package]] +name = "regex-lite" +version = "0.1.0" +dependencies = [ + "anyhow", + "regex-test", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "regex-test" +version = "0.1.0" +dependencies = [ + "anyhow", + "bstr", + "serde", + "toml", +] + +[[package]] +name = "rure" +version = "0.2.2" +dependencies = [ + "libc", + "regex", +] + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "2.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tabwriter" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08e1173ee641651a3095fe95d86ae314cd1f959888097debce3e0f9ca532eef1" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "termcolor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winnow" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +dependencies = [ + "memchr", +] diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain new file mode 100644 index 00000000000..96c467e091c --- /dev/null +++ b/compiler/rustc_codegen_cranelift/rust-toolchain @@ -0,0 +1,4 @@ +[toolchain] +channel = "nightly-2024-08-09" +components = ["rust-src", "rustc-dev", "llvm-tools"] +profile = "minimal" diff --git a/compiler/rustc_codegen_cranelift/rustfmt.toml b/compiler/rustc_codegen_cranelift/rustfmt.toml new file mode 100644 index 00000000000..d9e6ac3d543 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/rustfmt.toml @@ -0,0 +1,10 @@ +ignore = [ + "example/gen_block_iterate.rs", # uses edition 2024 +] + +# Matches rustfmt.toml of rustc +version = "Two" +use_small_heuristics = "Max" +merge_derives = false +group_imports = "StdExternalCrate" +imports_granularity = "Module" diff --git a/compiler/rustc_codegen_cranelift/scripts/Readme.md b/compiler/rustc_codegen_cranelift/scripts/Readme.md new file mode 100644 index 00000000000..83cec9c6f36 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/Readme.md @@ -0,0 +1,2 @@ +This directory is for scripts that are either never directly invoked or are not used very often. +Scripts that are frequently used should be kept at the project root. diff --git a/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs new file mode 100644 index 00000000000..1e14f41d4a2 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs @@ -0,0 +1,96 @@ +use std::env; +#[cfg(unix)] +use std::os::unix::process::CommandExt; +use std::process::Command; + +include!("../build_system/shared_utils.rs"); + +fn main() { + let current_exe = env::current_exe().unwrap(); + let mut sysroot = current_exe.parent().unwrap(); + if sysroot.file_name().unwrap().to_str().unwrap() == "bin" { + sysroot = sysroot.parent().unwrap(); + } + + let mut rustflags = vec!["-Cpanic=abort".to_owned(), "-Zpanic-abort-tests".to_owned()]; + if let Some(name) = option_env!("BUILTIN_BACKEND") { + rustflags.push(format!("-Zcodegen-backend={name}")); + } else { + let dylib = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join( + env::consts::DLL_PREFIX.to_string() + + "rustc_codegen_cranelift" + + env::consts::DLL_SUFFIX, + ); + rustflags.push(format!("-Zcodegen-backend={}", dylib.to_str().unwrap())); + } + rustflags.push("--sysroot".to_owned()); + rustflags.push(sysroot.to_str().unwrap().to_owned()); + + let cargo = if let Some(cargo) = option_env!("CARGO") { + cargo + } else { + // Ensure that the right toolchain is used + env::set_var("RUSTUP_TOOLCHAIN", option_env!("TOOLCHAIN_NAME").expect("TOOLCHAIN_NAME")); + "cargo" + }; + + let mut args = env::args().skip(1).collect::<Vec<_>>(); + if args.get(0).map(|arg| &**arg) == Some("clif") { + // Avoid infinite recursion when invoking `cargo-clif` as cargo subcommand using + // `cargo clif`. + args.remove(0); + } + + let args: Vec<_> = match args.get(0).map(|arg| &**arg) { + Some("jit") => { + rustflags.push("-Cprefer-dynamic".to_owned()); + args.remove(0); + IntoIterator::into_iter(["rustc".to_string()]) + .chain(args) + .chain([ + "--".to_string(), + "-Zunstable-options".to_string(), + "-Cllvm-args=mode=jit".to_string(), + ]) + .collect() + } + Some("lazy-jit") => { + rustflags.push("-Cprefer-dynamic".to_owned()); + args.remove(0); + IntoIterator::into_iter(["rustc".to_string()]) + .chain(args) + .chain([ + "--".to_string(), + "-Zunstable-options".to_string(), + "-Cllvm-args=mode=jit-lazy".to_string(), + ]) + .collect() + } + _ => args, + }; + + let mut cmd = Command::new(cargo); + cmd.args(args); + rustflags_to_cmd_env( + &mut cmd, + "RUSTFLAGS", + &rustflags_from_env("RUSTFLAGS") + .into_iter() + .chain(rustflags.iter().map(|flag| flag.clone())) + .collect::<Vec<_>>(), + ); + rustflags_to_cmd_env( + &mut cmd, + "RUSTDOCFLAGS", + &rustflags_from_env("RUSTDOCFLAGS") + .into_iter() + .chain(rustflags.iter().map(|flag| flag.clone())) + .collect::<Vec<_>>(), + ); + + #[cfg(unix)] + panic!("Failed to spawn cargo: {}", cmd.exec()); + + #[cfg(not(unix))] + std::process::exit(cmd.spawn().unwrap().wait().unwrap().code().unwrap_or(1)); +} diff --git a/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs b/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs new file mode 100755 index 00000000000..0252d5b3340 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +#![rustfmt::skip]/* This line is ignored by bash +# This block is ignored by rustc +pushd $(dirname "$0")/../ +RUSTC="$(pwd)/dist/rustc-clif" +popd +PROFILE=$1 OUTPUT=$2 exec $RUSTC -Zunstable-options -Cllvm-args=mode=jit -Cprefer-dynamic $0 +#*/ + +//! This program filters away uninteresting samples and trims uninteresting frames for stackcollapse +//! profiles. +//! +//! Usage: ./filter_profile.rs <profile in stackcollapse format> <output file> +//! +//! This file is specially crafted to be both a valid bash script and valid rust source file. If +//! executed as bash script this will run the rust source using cg_clif in JIT mode. + +use std::io::Write; + +fn main() -> Result<(), Box<dyn std::error::Error>> { + let profile_name = std::env::var("PROFILE").unwrap(); + let output_name = std::env::var("OUTPUT").unwrap(); + if profile_name.is_empty() || output_name.is_empty() { + println!("Usage: ./filter_profile.rs <profile in stackcollapse format> <output file>"); + std::process::exit(1); + } + let profile = std::fs::read_to_string(profile_name) + .map_err(|err| format!("Failed to read profile {}", err))?; + let mut output = + std::fs::OpenOptions::new().create(true).write(true).truncate(true).open(output_name)?; + + for line in profile.lines() { + let mut stack = &line[..line.rfind(" ").unwrap()]; + let count = &line[line.rfind(" ").unwrap() + 1..]; + + // Filter away uninteresting samples + if !stack.contains("rustc_codegen_cranelift") { + continue; + } + + if stack.contains("rustc_monomorphize::partitioning::collect_and_partition_mono_items") + || stack.contains("rustc_incremental::assert_dep_graph::assert_dep_graph") + || stack.contains("rustc_symbol_mangling::test::report_symbol_names") + { + continue; + } + + // Trim start + if let Some(index) = stack.find("rustc_interface::passes::configure_and_expand") { + stack = &stack[index..]; + } else if let Some(index) = stack.find("rustc_interface::passes::analysis") { + stack = &stack[index..]; + } else if let Some(index) = stack.find("rustc_interface::passes::start_codegen") { + stack = &stack[index..]; + } else if let Some(index) = stack.find("rustc_interface::queries::Linker::link") { + stack = &stack[index..]; + } + + if let Some(index) = stack.find("rustc_codegen_cranelift::driver::aot::module_codegen") { + stack = &stack[index..]; + } + + // Trim end + const MALLOC: &str = "malloc"; + if let Some(index) = stack.find(MALLOC) { + stack = &stack[..index + MALLOC.len()]; + } + + const FREE: &str = "free"; + if let Some(index) = stack.find(FREE) { + stack = &stack[..index + FREE.len()]; + } + + const TYPECK_ITEM_BODIES: &str = "rustc_typeck::check::typeck_item_bodies"; + if let Some(index) = stack.find(TYPECK_ITEM_BODIES) { + stack = &stack[..index + TYPECK_ITEM_BODIES.len()]; + } + + const COLLECT_AND_PARTITION_MONO_ITEMS: &str = + "rustc_monomorphize::partitioning::collect_and_partition_mono_items"; + if let Some(index) = stack.find(COLLECT_AND_PARTITION_MONO_ITEMS) { + stack = &stack[..index + COLLECT_AND_PARTITION_MONO_ITEMS.len()]; + } + + const ASSERT_DEP_GRAPH: &str = "rustc_incremental::assert_dep_graph::assert_dep_graph"; + if let Some(index) = stack.find(ASSERT_DEP_GRAPH) { + stack = &stack[..index + ASSERT_DEP_GRAPH.len()]; + } + + const REPORT_SYMBOL_NAMES: &str = "rustc_symbol_mangling::test::report_symbol_names"; + if let Some(index) = stack.find(REPORT_SYMBOL_NAMES) { + stack = &stack[..index + REPORT_SYMBOL_NAMES.len()]; + } + + const ENCODE_METADATA: &str = "rustc_metadata::rmeta::encoder::encode_metadata"; + if let Some(index) = stack.find(ENCODE_METADATA) { + stack = &stack[..index + ENCODE_METADATA.len()]; + } + + const INSTANTIATE_AND_NORMALIZE_ERASING_REGIONS: &str = "rustc_middle::ty::normalize_erasing_regions::<impl rustc_middle::ty::context::TyCtxt>::instantiate_and_normalize_erasing_regions"; + if let Some(index) = stack.find(INSTANTIATE_AND_NORMALIZE_ERASING_REGIONS) { + stack = &stack[..index + INSTANTIATE_AND_NORMALIZE_ERASING_REGIONS.len()]; + } + + const NORMALIZE_ERASING_LATE_BOUND_REGIONS: &str = "rustc_middle::ty::normalize_erasing_regions::<impl rustc_middle::ty::context::TyCtxt>::normalize_erasing_late_bound_regions"; + if let Some(index) = stack.find(NORMALIZE_ERASING_LATE_BOUND_REGIONS) { + stack = &stack[..index + NORMALIZE_ERASING_LATE_BOUND_REGIONS.len()]; + } + + const INST_BUILD: &str = "<cranelift_frontend::frontend::FuncInstBuilder as cranelift_codegen::ir::builder::InstBuilderBase>::build"; + if let Some(index) = stack.find(INST_BUILD) { + stack = &stack[..index + INST_BUILD.len()]; + } + + output.write_all(stack.as_bytes())?; + output.write_all(&*b" ")?; + output.write_all(count.as_bytes())?; + output.write_all(&*b"\n")?; + } + + Ok(()) +} diff --git a/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs b/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs new file mode 100644 index 00000000000..92defd21cd9 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs @@ -0,0 +1,53 @@ +use std::env; +use std::ffi::OsString; +#[cfg(unix)] +use std::os::unix::process::CommandExt; +use std::process::Command; + +fn main() { + let current_exe = env::current_exe().unwrap(); + let mut sysroot = current_exe.parent().unwrap(); + if sysroot.file_name().unwrap().to_str().unwrap() == "bin" { + sysroot = sysroot.parent().unwrap(); + } + + let cg_clif_dylib_path = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join( + env::consts::DLL_PREFIX.to_string() + "rustc_codegen_cranelift" + env::consts::DLL_SUFFIX, + ); + + let passed_args = std::env::args_os().skip(1).collect::<Vec<_>>(); + let mut args = vec![]; + args.push(OsString::from("-Cpanic=abort")); + args.push(OsString::from("-Zpanic-abort-tests")); + if let Some(name) = option_env!("BUILTIN_BACKEND") { + args.push(OsString::from(format!("-Zcodegen-backend={name}"))) + } else { + let mut codegen_backend_arg = OsString::from("-Zcodegen-backend="); + codegen_backend_arg.push(cg_clif_dylib_path); + args.push(codegen_backend_arg); + } + if !passed_args + .iter() + .any(|arg| arg == "--sysroot" || arg.to_str().is_some_and(|s| s.starts_with("--sysroot="))) + { + args.push(OsString::from("--sysroot")); + args.push(OsString::from(sysroot.to_str().unwrap())); + } + args.extend(passed_args); + + let rustc = if let Some(rustc) = option_env!("RUSTC") { + rustc + } else { + // Ensure that the right toolchain is used + env::set_var("RUSTUP_TOOLCHAIN", option_env!("TOOLCHAIN_NAME").expect("TOOLCHAIN_NAME")); + "rustc" + }; + + #[cfg(unix)] + panic!("Failed to spawn rustc: {}", Command::new(rustc).args(args).exec()); + + #[cfg(not(unix))] + std::process::exit( + Command::new(rustc).args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1), + ); +} diff --git a/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs b/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs new file mode 100644 index 00000000000..1cad312bb79 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs @@ -0,0 +1,58 @@ +use std::env; +use std::ffi::OsString; +#[cfg(unix)] +use std::os::unix::process::CommandExt; +use std::process::Command; + +fn main() { + let current_exe = env::current_exe().unwrap(); + let mut sysroot = current_exe.parent().unwrap(); + if sysroot.file_name().unwrap().to_str().unwrap() == "bin" { + sysroot = sysroot.parent().unwrap(); + } + + let cg_clif_dylib_path = sysroot.join(if cfg!(windows) { "bin" } else { "lib" }).join( + env::consts::DLL_PREFIX.to_string() + "rustc_codegen_cranelift" + env::consts::DLL_SUFFIX, + ); + + let passed_args = std::env::args_os().skip(1).collect::<Vec<_>>(); + let mut args = vec![]; + args.push(OsString::from("-Cpanic=abort")); + args.push(OsString::from("-Zpanic-abort-tests")); + if let Some(name) = option_env!("BUILTIN_BACKEND") { + args.push(OsString::from(format!("-Zcodegen-backend={name}"))) + } else { + let mut codegen_backend_arg = OsString::from("-Zcodegen-backend="); + codegen_backend_arg.push(cg_clif_dylib_path); + args.push(codegen_backend_arg); + } + if !passed_args + .iter() + .any(|arg| arg == "--sysroot" || arg.to_str().is_some_and(|s| s.starts_with("--sysroot="))) + { + args.push(OsString::from("--sysroot")); + args.push(OsString::from(sysroot.to_str().unwrap())); + } + if passed_args.is_empty() { + // Don't pass any arguments when the user didn't pass any arguments + // either to ensure the help message is shown. + args.clear(); + } + args.extend(passed_args); + + let rustdoc = if let Some(rustdoc) = option_env!("RUSTDOC") { + rustdoc + } else { + // Ensure that the right toolchain is used + env::set_var("RUSTUP_TOOLCHAIN", option_env!("TOOLCHAIN_NAME").expect("TOOLCHAIN_NAME")); + "rustdoc" + }; + + #[cfg(unix)] + panic!("Failed to spawn rustdoc: {}", Command::new(rustdoc).args(args).exec()); + + #[cfg(not(unix))] + std::process::exit( + Command::new(rustdoc).args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1), + ); +} diff --git a/compiler/rustc_codegen_cranelift/scripts/rustup.sh b/compiler/rustc_codegen_cranelift/scripts/rustup.sh new file mode 100755 index 00000000000..355282911c2 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/rustup.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -e + +TOOLCHAIN=${TOOLCHAIN:-$(date +%Y-%m-%d)} + +function check_git_fixed_subtree() { + if [[ ! -e ./git-fixed-subtree.sh ]]; then + echo "Missing git-fixed-subtree.sh. Please run the following commands to download it:" + echo "curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/bjorn3/git/tqc-subtree-portable/contrib/subtree/git-subtree.sh -o git-fixed-subtree.sh" + echo "chmod u+x git-fixed-subtree.sh" + exit 1 + fi + if [[ ! -x ./git-fixed-subtree.sh ]]; then + echo "git-fixed-subtree.sh is not executable. Please run the following command to make it executable:" + echo "chmod u+x git-fixed-subtree.sh" + exit 1 + fi +} + +case $1 in + "prepare") + echo "=> Installing new nightly" + rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists + sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain + rustup component add rustfmt || true + + echo "=> Uninstalling all old nightlies" + for nightly in $(rustup toolchain list | grep nightly | grep -v "$TOOLCHAIN" | grep -v nightly-x86_64); do + rustup toolchain uninstall "$nightly" + done + + ./clean_all.sh + + ./y.sh prepare + ;; + "commit") + git add rust-toolchain + git commit -m "Rustup to $(rustc -V)" + ;; + "push") + check_git_fixed_subtree + + cg_clif=$(pwd) + pushd ../rust + git pull origin master + branch=sync_cg_clif-$(date +%Y-%m-%d) + git checkout -b "$branch" + "$cg_clif/git-fixed-subtree.sh" pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/rust-lang/rustc_codegen_cranelift.git master + git push -u my "$branch" + + # immediately merge the merge commit into cg_clif to prevent merge conflicts when syncing + # from rust-lang/rust later + "$cg_clif/git-fixed-subtree.sh" push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust + popd + git merge sync_from_rust + ;; + "pull") + check_git_fixed_subtree + + RUST_VERS=$(curl "https://static.rust-lang.org/dist/$TOOLCHAIN/channel-rust-nightly-git-commit-hash.txt") + echo "Pulling $RUST_VERS ($TOOLCHAIN)" + + cg_clif=$(pwd) + pushd ../rust + git fetch origin master + git checkout "$RUST_VERS" + "$cg_clif/git-fixed-subtree.sh" push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust + popd + git merge sync_from_rust -m "Sync from rust $RUST_VERS" + git branch -d sync_from_rust + ;; + *) + echo "Unknown command '$1'" + echo "Usage: ./rustup.sh prepare|commit" + ;; +esac diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh new file mode 100644 index 00000000000..684a5d07293 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -e + +# CG_CLIF_FORCE_GNU_AS will force usage of as instead of the LLVM backend of rustc as +# the LLVM backend isn't compiled in here. +export CG_CLIF_FORCE_GNU_AS=1 + +# Compiletest expects all standard library paths to start with /rustc/FAKE_PREFIX. +# CG_CLIF_STDLIB_REMAP_PATH_PREFIX will cause cg_clif's build system to pass +# --remap-path-prefix to handle this. +CG_CLIF_STDLIB_REMAP_PATH_PREFIX=/rustc/FAKE_PREFIX ./y.sh build + +echo "[SETUP] Rust fork" +git clone --quiet https://github.com/rust-lang/rust.git --filter=tree:0 || true +pushd rust +git fetch +git checkout --no-progress -- . +git checkout --no-progress "$(rustc -V | cut -d' ' -f3 | tr -d '(')" + +git submodule update --quiet --init src/tools/cargo library/backtrace library/stdarch + +git -c user.name=Dummy -c user.email=dummy@example.com -c commit.gpgSign=false \ + am ../patches/*-stdlib-*.patch + +cat > config.toml <<EOF +change-id = 999999 + +[build] +rustc = "$(pwd)/../dist/bin/rustc-clif" +cargo = "$(rustup which cargo)" +full-bootstrap = true +local-rebuild = true + +[rust] +codegen-backends = ["cranelift"] +deny-warnings = false +verbose-tests = false +EOF +popd + +# Allow the testsuite to use llvm tools +host_triple=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") +export LLVM_BIN_DIR="$(rustc --print sysroot)/lib/rustlib/$host_triple/bin" diff --git a/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh b/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh new file mode 100755 index 00000000000..791d457993d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e + +cd "$(dirname "$0")/../" + +source ./scripts/setup_rust_fork.sh + +echo "[TEST] Bootstrap of rustc" +pushd rust +rm -r compiler/rustc_codegen_cranelift/{Cargo.*,src} +cp ../Cargo.* compiler/rustc_codegen_cranelift/ +cp -r ../src compiler/rustc_codegen_cranelift/src + +./x.py build --stage 1 library/std +popd diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh new file mode 100755 index 00000000000..bb5af9127b9 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +set -e + +cd $(dirname "$0")/../ + +source ./scripts/setup_rust_fork.sh + +echo "[TEST] Test suite of rustc" +pushd rust + +command -v rg >/dev/null 2>&1 || cargo install ripgrep + +rm -r tests/ui/{unsized-locals/,lto/,linkage*} || true +for test in $(rg --files-with-matches "lto" tests/{codegen-units,ui,incremental}); do + rm $test +done + +# should-fail tests don't work when compiletest is compiled with panic=abort +for test in $(rg --files-with-matches "//@ should-fail" tests/{codegen-units,ui,incremental}); do + rm $test +done + +for test in $(rg -i --files-with-matches "//(\[\w+\])?~[^\|]*\s*ERR|//@ error-pattern:|//@(\[.*\])? build-fail|//@(\[.*\])? run-fail|-Cllvm-args" tests/ui); do + rm $test +done + +git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed +git checkout -- tests/ui/proc-macro/pretty-print-hack/ +git checkout -- tests/ui/entry-point/auxiliary/bad_main_functions.rs +rm tests/ui/parser/unclosed-delimiter-in-dep.rs # submodule contains //~ERROR + +# missing features +# ================ + +# vendor intrinsics +rm tests/ui/asm/x86_64/evex512-implicit-feature.rs # unimplemented AVX512 x86 vendor intrinsic +rm tests/ui/simd/dont-invalid-bitcast-x86_64.rs # unimplemented llvm.x86.sse41.round.ps + +# exotic linkages +rm tests/incremental/hashes/function_interfaces.rs +rm tests/incremental/hashes/statics.rs +rm -r tests/run-make/naked-symbol-visibility + +# variadic arguments +rm tests/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs +rm tests/ui/abi/variadic-ffi.rs # requires callee side vararg support +rm -r tests/run-make/c-link-to-rust-va-list-fn # requires callee side vararg support +rm tests/ui/delegation/fn-header.rs + +# unsized locals +rm -r tests/run-pass-valgrind/unsized-locals + +# misc unimplemented things +rm tests/ui/target-feature/missing-plusminus.rs # error not implemented +rm -r tests/run-make/repr128-dwarf # debuginfo test +rm -r tests/run-make/split-debuginfo # same +rm -r tests/run-make/target-specs # i686 not supported by Cranelift +rm -r tests/run-make/mismatching-target-triples # same +rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly +rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported +rm tests/ui/simd/simd-bitmask-notpow2.rs # non-pow-of-2 simd vector sizes + +# requires LTO +rm -r tests/run-make/cdylib +rm -r tests/run-make/codegen-options-parsing +rm -r tests/run-make/lto-* +rm -r tests/run-make/reproducible-build-2 +rm -r tests/run-make/no-builtins-lto +rm -r tests/run-make/reachable-extern-fn-available-lto + +# coverage instrumentation +rm tests/ui/consts/precise-drop-with-coverage.rs +rm tests/ui/issues/issue-85461.rs +rm -r tests/ui/instrument-coverage/ + +# missing f16/f128 support +rm tests/ui/half-open-range-patterns/half-open-range-pats-semantics.rs + +# optimization tests +# ================== +rm tests/ui/codegen/issue-28950.rs # depends on stack size optimizations +rm tests/ui/codegen/init-large-type.rs # same +rm tests/ui/issues/issue-40883.rs # same +rm -r tests/run-make/fmt-write-bloat/ # tests an optimization +rm tests/ui/statics/const_generics.rs # same + +# backend specific tests +# ====================== +rm tests/incremental/thinlto/cgu_invalidated_when_import_{added,removed}.rs # requires LLVM +rm -r tests/run-make/cross-lang-lto # same +rm -r tests/run-make/sepcomp-inlining # same +rm -r tests/run-make/sepcomp-separate # same +rm -r tests/run-make/sepcomp-cci-copies # same +rm -r tests/run-make/volatile-intrinsics # same +rm -r tests/run-make/llvm-ident # same +rm -r tests/run-make/no-builtins-attribute # same +rm -r tests/run-make/pgo-gen-no-imp-symbols # same +rm tests/ui/abi/stack-protector.rs # requires stack protector support +rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes +rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific +rm -r tests/run-make/print-to-output # requires --print relocation-models + +# requires asm, llvm-ir and/or llvm-bc emit support +# ============================================= +rm -r tests/run-make/emit-named-files +rm -r tests/run-make/multiple-emits +rm -r tests/run-make/output-type-permutations +rm -r tests/run-make/emit-to-stdout +rm -r tests/run-make/compressed-debuginfo +rm -r tests/run-make/symbols-include-type-name +rm -r tests/run-make/notify-all-emit-artifacts +rm -r tests/run-make/reset-codegen-1 +rm -r tests/run-make/inline-always-many-cgu +rm -r tests/run-make/intrinsic-unreachable + +# giving different but possibly correct results +# ============================================= +rm tests/ui/mir/mir_misc_casts.rs # depends on deduplication of constants +rm tests/ui/mir/mir_raw_fat_ptr.rs # same +rm tests/ui/consts/issue-33537.rs # same +rm tests/ui/consts/const-mut-refs-crate.rs # same +rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift + +# doesn't work due to the way the rustc test suite is invoked. +# should work when using ./x.py test the way it is intended +# ============================================================ +rm -r tests/run-make/remap-path-prefix-dwarf # requires llvm-dwarfdump +rm -r tests/run-make/compiler-builtins # Expects lib/rustlib/src/rust to contains the standard library source + +# genuine bugs +# ============ +rm -r tests/run-make/extern-fn-explicit-align # argument alignment not yet supported +rm -r tests/run-make/panic-abort-eh_frame # .eh_frame emitted with panic=abort +rm tests/ui/deprecation/deprecated_inline_threshold.rs # missing deprecation warning for -Cinline-threshold + +# bugs in the test suite +# ====================== +rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue +rm tests/ui/backtrace/synchronized-panic-handler.rs # missing needs-unwind annotation +rm -r tests/ui/codegen/equal-pointers-unequal # make incorrect assumptions about the location of stack variables + +rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd + +cp ../dist/bin/rustdoc-clif ../dist/bin/rustdoc # some tests expect bin/rustdoc to exist + +# prevent $(RUSTDOC) from picking up the sysroot built by x.py. It conflicts with the one used by +# rustdoc-clif +cat <<EOF | git apply - +diff --git a/tests/run-make/tools.mk b/tests/run-make/tools.mk +index ea06b620c4c..b969d0009c6 100644 +--- a/tests/run-make/tools.mk ++++ b/tests/run-make/tools.mk +@@ -9,7 +9,7 @@ RUSTC_ORIGINAL := \$(RUSTC) + BARE_RUSTC := \$(HOST_RPATH_ENV) '\$(RUSTC)' + BARE_RUSTDOC := \$(HOST_RPATH_ENV) '\$(RUSTDOC)' + RUSTC := \$(BARE_RUSTC) --out-dir \$(TMPDIR) -L \$(TMPDIR) \$(RUSTFLAGS) -Ainternal_features +-RUSTDOC := \$(BARE_RUSTDOC) -L \$(TARGET_RPATH_DIR) ++RUSTDOC := \$(BARE_RUSTDOC) + ifdef RUSTC_LINKER + RUSTC := \$(RUSTC) -Clinker='\$(RUSTC_LINKER)' + RUSTDOC := \$(RUSTDOC) -Clinker='\$(RUSTC_LINKER)' +diff --git a/src/tools/run-make-support/src/rustdoc.rs b/src/tools/run-make-support/src/rustdoc.rs +index 9607ff02f96..b7d97caf9a2 100644 +--- a/src/tools/run-make-support/src/external_deps/rustdoc.rs ++++ b/src/tools/run-make-support/src/external_deps/rustdoc.rs +@@ -34,8 +34,6 @@ pub fn bare() -> Self { + #[track_caller] + pub fn new() -> Self { + let mut cmd = setup_common(); +- let target_rpath_dir = env_var_os("TARGET_RPATH_DIR"); +- cmd.arg(format!("-L{}", target_rpath_dir.to_string_lossy())); + Self { cmd } + } + +EOF + +echo "[TEST] rustc test suite" +COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 --test-args=--nocapture tests/{codegen-units,run-make,run-pass-valgrind,ui,incremental} +popd diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs new file mode 100644 index 00000000000..a318cae1722 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs @@ -0,0 +1,97 @@ +//! Annotate the clif ir with comments describing how arguments are passed into the current function +//! and where all locals are stored. + +use std::borrow::Cow; + +use rustc_target::abi::call::PassMode; + +use crate::prelude::*; + +pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) { + if fx.clif_comments.enabled() { + fx.add_global_comment( + "kind loc.idx param pass mode ty".to_string(), + ); + } +} + +pub(super) fn add_arg_comment<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + kind: &str, + local: Option<mir::Local>, + local_field: Option<usize>, + params: &[Value], + arg_abi_mode: &PassMode, + arg_layout: TyAndLayout<'tcx>, +) { + if !fx.clif_comments.enabled() { + return; + } + + let local = if let Some(local) = local { + Cow::Owned(format!("{:?}", local)) + } else { + Cow::Borrowed("???") + }; + let local_field = if let Some(local_field) = local_field { + Cow::Owned(format!(".{}", local_field)) + } else { + Cow::Borrowed("") + }; + + let params = match params { + [] => Cow::Borrowed("-"), + [param] => Cow::Owned(format!("= {:?}", param)), + [param_a, param_b] => Cow::Owned(format!("= {:?},{:?}", param_a, param_b)), + params => Cow::Owned(format!( + "= {}", + params.iter().map(ToString::to_string).collect::<Vec<_>>().join(",") + )), + }; + + let pass_mode = format!("{:?}", arg_abi_mode); + fx.add_global_comment(format!( + "{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}", + kind = kind, + local = local, + local_field = local_field, + params = params, + pass_mode = pass_mode, + ty = arg_layout.ty, + )); +} + +pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) { + if fx.clif_comments.enabled() { + fx.add_global_comment(String::new()); + fx.add_global_comment( + "kind local ty size align (abi,pref)".to_string(), + ); + } +} + +pub(super) fn add_local_place_comments<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + place: CPlace<'tcx>, + local: Local, +) { + if !fx.clif_comments.enabled() { + return; + } + let TyAndLayout { ty, layout } = place.layout(); + let rustc_target::abi::LayoutS { size, align, .. } = layout.0.0; + + let (kind, extra) = place.debug_comment(); + + fx.add_global_comment(format!( + "{:<5} {:5} {:30} {:4}b {}, {}{}{}", + kind, + format!("{:?}", local), + format!("{:?}", ty), + size.bytes(), + align.abi.bytes(), + align.pref.bytes(), + if extra.is_empty() { "" } else { " " }, + extra, + )); +} diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs new file mode 100644 index 00000000000..24cf3f061a5 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -0,0 +1,793 @@ +//! Handling of everything related to the calling convention. Also fills `fx.local_map`. + +mod comments; +mod pass_mode; +mod returning; + +use std::borrow::Cow; +use std::mem; + +use cranelift_codegen::ir::{ArgumentPurpose, SigRef}; +use cranelift_codegen::isa::CallConv; +use cranelift_module::ModuleError; +use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; +use rustc_codegen_ssa::errors::CompilerBuiltinsCannotCall; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::ty::TypeVisitableExt; +use rustc_session::Session; +use rustc_span::source_map::Spanned; +use rustc_target::abi::call::{Conv, FnAbi, PassMode}; +use rustc_target::spec::abi::Abi; + +use self::pass_mode::*; +pub(crate) use self::returning::codegen_return; +use crate::prelude::*; + +fn clif_sig_from_fn_abi<'tcx>( + tcx: TyCtxt<'tcx>, + default_call_conv: CallConv, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, +) -> Signature { + let call_conv = conv_to_call_conv(tcx.sess, fn_abi.conv, default_call_conv); + + let inputs = fn_abi.args.iter().flat_map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()); + + let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx); + // Sometimes the first param is a pointer to the place where the return value needs to be stored. + let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect(); + + Signature { params, returns, call_conv } +} + +pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv { + match c { + Conv::Rust | Conv::C => default_call_conv, + Conv::Cold | Conv::PreserveMost | Conv::PreserveAll => CallConv::Cold, + Conv::X86_64SysV => CallConv::SystemV, + Conv::X86_64Win64 => CallConv::WindowsFastcall, + + // Should already get a back compat warning + Conv::X86Fastcall | Conv::X86Stdcall | Conv::X86ThisCall | Conv::X86VectorCall => { + default_call_conv + } + + Conv::X86Intr | Conv::RiscvInterrupt { .. } => { + sess.dcx().fatal(format!("interrupt call conv {c:?} not yet implemented")) + } + + Conv::ArmAapcs => sess.dcx().fatal("aapcs call conv not yet implemented"), + Conv::CCmseNonSecureCall => { + sess.dcx().fatal("C-cmse-nonsecure-call call conv is not yet implemented"); + } + + Conv::Msp430Intr | Conv::PtxKernel | Conv::AvrInterrupt | Conv::AvrNonBlockingInterrupt => { + unreachable!("tried to use {c:?} call conv which only exists on an unsupported target"); + } + } +} + +pub(crate) fn get_function_sig<'tcx>( + tcx: TyCtxt<'tcx>, + default_call_conv: CallConv, + inst: Instance<'tcx>, +) -> Signature { + assert!(!inst.args.has_infer()); + clif_sig_from_fn_abi( + tcx, + default_call_conv, + &RevealAllLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()), + ) +} + +/// Instance must be monomorphized +pub(crate) fn import_function<'tcx>( + tcx: TyCtxt<'tcx>, + module: &mut dyn Module, + inst: Instance<'tcx>, +) -> FuncId { + let name = tcx.symbol_name(inst).name; + let sig = get_function_sig(tcx, module.target_config().default_call_conv, inst); + match module.declare_function(name, Linkage::Import, &sig) { + Ok(func_id) => func_id, + Err(ModuleError::IncompatibleDeclaration(_)) => tcx.dcx().fatal(format!( + "attempt to declare `{name}` as function, but it was already declared as static" + )), + Err(ModuleError::IncompatibleSignature(_, prev_sig, new_sig)) => tcx.dcx().fatal(format!( + "attempt to declare `{name}` with signature {new_sig:?}, \ + but it was already declared with signature {prev_sig:?}" + )), + Err(err) => Err::<_, _>(err).unwrap(), + } +} + +impl<'tcx> FunctionCx<'_, '_, 'tcx> { + /// Instance must be monomorphized + pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef { + let func_id = import_function(self.tcx, self.module, inst); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); + + if self.clif_comments.enabled() { + self.add_comment(func_ref, format!("{:?}", inst)); + } + + func_ref + } + + pub(crate) fn lib_call( + &mut self, + name: &str, + params: Vec<AbiParam>, + returns: Vec<AbiParam>, + args: &[Value], + ) -> Cow<'_, [Value]> { + if self.tcx.sess.target.is_like_windows { + let (mut params, mut args): (Vec<_>, Vec<_>) = params + .into_iter() + .zip(args) + .map(|(param, &arg)| { + if param.value_type == types::I128 { + let arg_ptr = self.create_stack_slot(16, 16); + arg_ptr.store(self, arg, MemFlags::trusted()); + (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self)) + } else { + (param, arg) + } + }) + .unzip(); + + let indirect_ret_val = returns.len() == 1 && returns[0].value_type == types::I128; + + if indirect_ret_val { + params.insert(0, AbiParam::new(self.pointer_type)); + let ret_ptr = self.create_stack_slot(16, 16); + args.insert(0, ret_ptr.get_addr(self)); + self.lib_call_unadjusted(name, params, vec![], &args); + return Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]); + } else { + return self.lib_call_unadjusted(name, params, returns, &args); + } + } + + self.lib_call_unadjusted(name, params, returns, args) + } + + pub(crate) fn lib_call_unadjusted( + &mut self, + name: &str, + params: Vec<AbiParam>, + returns: Vec<AbiParam>, + args: &[Value], + ) -> Cow<'_, [Value]> { + let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv }; + let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap(); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); + if self.clif_comments.enabled() { + self.add_comment(func_ref, format!("{:?}", name)); + } + let call_inst = self.bcx.ins().call(func_ref, args); + if self.clif_comments.enabled() { + self.add_comment(call_inst, format!("lib_call {}", name)); + } + let results = self.bcx.inst_results(call_inst); + assert!(results.len() <= 2, "{}", results.len()); + Cow::Borrowed(results) + } +} + +/// Make a [`CPlace`] capable of holding value of the specified type. +fn make_local_place<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + local: Local, + layout: TyAndLayout<'tcx>, + is_ssa: bool, +) -> CPlace<'tcx> { + if layout.is_unsized() { + fx.tcx.dcx().span_fatal( + fx.mir.local_decls[local].source_info.span, + "unsized locals are not yet supported", + ); + } + let place = if is_ssa { + if let rustc_target::abi::Abi::ScalarPair(_, _) = layout.abi { + CPlace::new_var_pair(fx, local, layout) + } else { + CPlace::new_var(fx, local, layout) + } + } else { + CPlace::new_stack_slot(fx, layout) + }; + + self::comments::add_local_place_comments(fx, place, local); + + place +} + +pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_block: Block) { + fx.bcx.append_block_params_for_function_params(start_block); + + fx.bcx.switch_to_block(start_block); + fx.bcx.ins().nop(); + + let ssa_analyzed = crate::analyze::analyze(fx); + + self::comments::add_args_header_comment(fx); + + let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter(); + let ret_place = + self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); + assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); + + // None means pass_mode == NoPass + enum ArgKind<'tcx> { + Normal(Option<CValue<'tcx>>), + Spread(Vec<Option<CValue<'tcx>>>), + } + + // FIXME implement variadics in cranelift + if fx.fn_abi.c_variadic { + fx.tcx.dcx().span_fatal( + fx.mir.span, + "Defining variadic functions is not yet supported by Cranelift", + ); + } + + let mut arg_abis_iter = fx.fn_abi.args.iter(); + + let func_params = fx + .mir + .args_iter() + .map(|local| { + let arg_ty = fx.monomorphize(fx.mir.local_decls[local].ty); + + // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482 + if Some(local) == fx.mir.spread_arg { + // This argument (e.g. the last argument in the "rust-call" ABI) + // is a tuple that was spread at the ABI level and now we have + // to reconstruct it into a tuple local variable, from multiple + // individual function arguments. + + let tupled_arg_tys = match arg_ty.kind() { + ty::Tuple(ref tys) => tys, + _ => bug!("spread argument isn't a tuple?! but {:?}", arg_ty), + }; + + let mut params = Vec::new(); + for (i, _arg_ty) in tupled_arg_tys.iter().enumerate() { + let arg_abi = arg_abis_iter.next().unwrap(); + let param = + cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); + params.push(param); + } + + (local, ArgKind::Spread(params), arg_ty) + } else { + let arg_abi = arg_abis_iter.next().unwrap(); + let param = + cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); + (local, ArgKind::Normal(param), arg_ty) + } + }) + .collect::<Vec<(Local, ArgKind<'tcx>, Ty<'tcx>)>>(); + + assert!(fx.caller_location.is_none()); + if fx.instance.def.requires_caller_location(fx.tcx) { + // Store caller location for `#[track_caller]`. + let arg_abi = arg_abis_iter.next().unwrap(); + fx.caller_location = + Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap()); + } + + assert!(arg_abis_iter.next().is_none(), "ArgAbi left behind"); + assert!(block_params_iter.next().is_none(), "arg_value left behind"); + + self::comments::add_locals_header_comment(fx); + + for (local, arg_kind, ty) in func_params { + // While this is normally an optimization to prevent an unnecessary copy when an argument is + // not mutated by the current function, this is necessary to support unsized arguments. + if let ArgKind::Normal(Some(val)) = arg_kind { + if let Some((addr, meta)) = val.try_to_ptr() { + // Ownership of the value at the backing storage for an argument is passed to the + // callee per the ABI, so it is fine to borrow the backing storage of this argument + // to prevent a copy. + + let place = if let Some(meta) = meta { + CPlace::for_ptr_with_extra(addr, meta, val.layout()) + } else { + CPlace::for_ptr(addr, val.layout()) + }; + + self::comments::add_local_place_comments(fx, place, local); + + assert_eq!(fx.local_map.push(place), local); + continue; + } + } + + let layout = fx.layout_of(ty); + let is_ssa = ssa_analyzed[local].is_ssa(fx, ty); + let place = make_local_place(fx, local, layout, is_ssa); + assert_eq!(fx.local_map.push(place), local); + + match arg_kind { + ArgKind::Normal(param) => { + if let Some(param) = param { + place.write_cvalue(fx, param); + } + } + ArgKind::Spread(params) => { + for (i, param) in params.into_iter().enumerate() { + if let Some(param) = param { + place.place_field(fx, FieldIdx::new(i)).write_cvalue(fx, param); + } + } + } + } + } + + for local in fx.mir.vars_and_temps_iter() { + let ty = fx.monomorphize(fx.mir.local_decls[local].ty); + let layout = fx.layout_of(ty); + + let is_ssa = ssa_analyzed[local].is_ssa(fx, ty); + + let place = make_local_place(fx, local, layout, is_ssa); + assert_eq!(fx.local_map.push(place), local); + } + + fx.bcx.ins().jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]); +} + +struct CallArgument<'tcx> { + value: CValue<'tcx>, + is_owned: bool, +} + +// FIXME avoid intermediate `CValue` before calling `adjust_arg_for_abi` +fn codegen_call_argument_operand<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + operand: &Operand<'tcx>, +) -> CallArgument<'tcx> { + CallArgument { + value: codegen_operand(fx, operand), + is_owned: matches!(operand, Operand::Move(_)), + } +} + +pub(crate) fn codegen_terminator_call<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + source_info: mir::SourceInfo, + func: &Operand<'tcx>, + args: &[Spanned<Operand<'tcx>>], + destination: Place<'tcx>, + target: Option<BasicBlock>, +) { + let func = codegen_operand(fx, func); + let fn_sig = func.layout().ty.fn_sig(fx.tcx); + + let ret_place = codegen_place(fx, destination); + + // Handle special calls like intrinsics and empty drop glue. + let instance = if let ty::FnDef(def_id, fn_args) = *func.layout().ty.kind() { + let instance = ty::Instance::expect_resolve( + fx.tcx, + ty::ParamEnv::reveal_all(), + def_id, + fn_args, + source_info.span, + ) + .polymorphize(fx.tcx); + + if is_call_from_compiler_builtins_to_upstream_monomorphization(fx.tcx, instance) { + if target.is_some() { + let caller = with_no_trimmed_paths!(fx.tcx.def_path_str(fx.instance.def_id())); + let callee = with_no_trimmed_paths!(fx.tcx.def_path_str(def_id)); + fx.tcx.dcx().emit_err(CompilerBuiltinsCannotCall { caller, callee }); + } else { + fx.bcx.ins().trap(TrapCode::User(0)); + return; + } + } + + if fx.tcx.symbol_name(instance).name.starts_with("llvm.") { + crate::intrinsics::codegen_llvm_intrinsic_call( + fx, + &fx.tcx.symbol_name(instance).name, + args, + ret_place, + target, + source_info.span, + ); + return; + } + + match instance.def { + InstanceKind::Intrinsic(_) => { + match crate::intrinsics::codegen_intrinsic_call( + fx, + instance, + args, + ret_place, + target, + source_info, + ) { + Ok(()) => return, + Err(instance) => Some(instance), + } + } + InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) => { + // empty drop glue - a nop. + let dest = target.expect("Non terminating drop_in_place_real???"); + let ret_block = fx.get_block(dest); + fx.bcx.ins().jump(ret_block, &[]); + return; + } + _ => Some(instance), + } + } else { + None + }; + + let extra_args = &args[fn_sig.inputs().skip_binder().len()..]; + let extra_args = fx.tcx.mk_type_list_from_iter( + extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.node.ty(fx.mir, fx.tcx))), + ); + let fn_abi = if let Some(instance) = instance { + RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args) + } else { + RevealAllLayoutCx(fx.tcx).fn_abi_of_fn_ptr(fn_sig, extra_args) + }; + + let is_cold = if fn_sig.abi() == Abi::RustCold { + true + } else { + instance.is_some_and(|inst| { + fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD) + }) + }; + if is_cold { + fx.bcx.set_cold_block(fx.bcx.current_block().unwrap()); + if let Some(destination_block) = target { + fx.bcx.set_cold_block(fx.get_block(destination_block)); + } + } + + // Unpack arguments tuple for closures + let mut args = if fn_sig.abi() == Abi::RustCall { + let (self_arg, pack_arg) = match args { + [pack_arg] => (None, codegen_call_argument_operand(fx, &pack_arg.node)), + [self_arg, pack_arg] => ( + Some(codegen_call_argument_operand(fx, &self_arg.node)), + codegen_call_argument_operand(fx, &pack_arg.node), + ), + _ => panic!("rust-call abi requires one or two arguments"), + }; + + let tupled_arguments = match pack_arg.value.layout().ty.kind() { + ty::Tuple(ref tupled_arguments) => tupled_arguments, + _ => bug!("argument to function with \"rust-call\" ABI is not a tuple"), + }; + + let mut args = Vec::with_capacity(1 + tupled_arguments.len()); + args.extend(self_arg); + for i in 0..tupled_arguments.len() { + args.push(CallArgument { + value: pack_arg.value.value_field(fx, FieldIdx::new(i)), + is_owned: pack_arg.is_owned, + }); + } + args + } else { + args.iter().map(|arg| codegen_call_argument_operand(fx, &arg.node)).collect::<Vec<_>>() + }; + + // Pass the caller location for `#[track_caller]`. + if instance.is_some_and(|inst| inst.def.requires_caller_location(fx.tcx)) { + let caller_location = fx.get_caller_location(source_info); + args.push(CallArgument { value: caller_location, is_owned: false }); + } + + let args = args; + assert_eq!(fn_abi.args.len(), args.len()); + + #[derive(Copy, Clone)] + enum CallTarget { + Direct(FuncRef), + Indirect(SigRef, Value), + } + + let (func_ref, first_arg_override) = match instance { + // Trait object call + Some(Instance { def: InstanceKind::Virtual(_, idx), .. }) => { + if fx.clif_comments.enabled() { + let nop_inst = fx.bcx.ins().nop(); + fx.add_comment( + nop_inst, + format!("virtual call; self arg pass mode: {:?}", fn_abi.args[0]), + ); + } + + let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0].value, idx); + let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); + let sig = fx.bcx.import_signature(sig); + + (CallTarget::Indirect(sig, method), Some(ptr.get_addr(fx))) + } + + // Normal call + Some(instance) => { + let func_ref = fx.get_function_ref(instance); + (CallTarget::Direct(func_ref), None) + } + + // Indirect call + None => { + if fx.clif_comments.enabled() { + let nop_inst = fx.bcx.ins().nop(); + fx.add_comment(nop_inst, "indirect call"); + } + + let func = func.load_scalar(fx); + let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); + let sig = fx.bcx.import_signature(sig); + + (CallTarget::Indirect(sig, func), None) + } + }; + + self::returning::codegen_with_call_return_arg(fx, &fn_abi.ret, ret_place, |fx, return_ptr| { + let mut call_args = return_ptr + .into_iter() + .chain(first_arg_override.into_iter()) + .chain( + args.into_iter() + .enumerate() + .skip(if first_arg_override.is_some() { 1 } else { 0 }) + .flat_map(|(i, arg)| { + adjust_arg_for_abi(fx, arg.value, &fn_abi.args[i], arg.is_owned).into_iter() + }), + ) + .collect::<Vec<Value>>(); + + // FIXME: Find a cleaner way to support varargs. + if fn_abi.c_variadic { + adjust_call_for_c_variadic(fx, &fn_abi, source_info, func_ref, &mut call_args); + } + + match func_ref { + CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args), + CallTarget::Indirect(sig, func_ptr) => { + fx.bcx.ins().call_indirect(sig, func_ptr, &call_args) + } + } + }); + + if let Some(dest) = target { + let ret_block = fx.get_block(dest); + fx.bcx.ins().jump(ret_block, &[]); + } else { + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + } + + fn adjust_call_for_c_variadic<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + source_info: mir::SourceInfo, + target: CallTarget, + call_args: &mut Vec<Value>, + ) { + if fn_abi.conv != Conv::C { + fx.tcx.dcx().span_fatal( + source_info.span, + format!("Variadic call for non-C abi {:?}", fn_abi.conv), + ); + } + let sig_ref = match target { + CallTarget::Direct(func_ref) => fx.bcx.func.dfg.ext_funcs[func_ref].signature, + CallTarget::Indirect(sig_ref, _) => sig_ref, + }; + // `mem::take()` the `params` so that `fx.bcx` can be used below. + let mut abi_params = mem::take(&mut fx.bcx.func.dfg.signatures[sig_ref].params); + + // Recalculate the parameters in the signature to ensure the signature contains the variadic arguments. + let has_return_arg = matches!(fn_abi.ret.mode, PassMode::Indirect { .. }); + // Drop everything except the return argument (if there is one). + abi_params.truncate(if has_return_arg { 1 } else { 0 }); + // Add the fixed arguments. + abi_params.extend( + fn_abi.args[..fn_abi.fixed_count as usize] + .iter() + .flat_map(|arg_abi| arg_abi.get_abi_param(fx.tcx).into_iter()), + ); + let fixed_arg_count = abi_params.len(); + // Add the variadic arguments. + abi_params.extend( + fn_abi.args[fn_abi.fixed_count as usize..] + .iter() + .flat_map(|arg_abi| arg_abi.get_abi_param(fx.tcx).into_iter()), + ); + + if fx.tcx.sess.target.is_like_osx && fx.tcx.sess.target.arch == "aarch64" { + // Add any padding arguments needed for Apple AArch64. + // There's no need to pad the argument list unless variadic arguments are actually being + // passed. + if abi_params.len() > fixed_arg_count { + // 128-bit integers take 2 registers, and everything else takes 1. + // FIXME: Add support for non-integer types + // This relies on the checks below to ensure all arguments are integer types and + // that the ABI is "C". + // The return argument isn't counted as it goes in its own dedicated register. + let integer_registers_used: usize = abi_params + [if has_return_arg { 1 } else { 0 }..fixed_arg_count] + .iter() + .map(|arg| if arg.value_type.bits() == 128 { 2 } else { 1 }) + .sum(); + // The ABI uses 8 registers before it starts pushing arguments to the stack. Pad out + // the registers if needed to ensure the variadic arguments are passed on the stack. + if integer_registers_used < 8 { + abi_params.splice( + fixed_arg_count..fixed_arg_count, + (integer_registers_used..8).map(|_| AbiParam::new(types::I64)), + ); + call_args.splice( + fixed_arg_count..fixed_arg_count, + (integer_registers_used..8).map(|_| fx.bcx.ins().iconst(types::I64, 0)), + ); + } + } + + // `StructArgument` is not currently used by the `aarch64` ABI, and is therefore not + // handled when calculating how many padding arguments to use. Assert that this remains + // the case. + assert!(abi_params.iter().all(|param| matches!( + param.purpose, + // The only purposes used are `Normal` and `StructReturn`. + ArgumentPurpose::Normal | ArgumentPurpose::StructReturn + ))); + } + + // Check all parameters are integers. + for param in abi_params.iter() { + if !param.value_type.is_int() { + // FIXME: Set %al to upperbound on float args once floats are supported. + fx.tcx.dcx().span_fatal( + source_info.span, + format!("Non int ty {:?} for variadic call", param.value_type), + ); + } + } + + assert_eq!(abi_params.len(), call_args.len()); + + // Put the `AbiParam`s back in the signature. + fx.bcx.func.dfg.signatures[sig_ref].params = abi_params; + } +} + +pub(crate) fn codegen_drop<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + source_info: mir::SourceInfo, + drop_place: CPlace<'tcx>, + target: BasicBlock, +) { + let ty = drop_place.layout().ty; + let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx); + + if let ty::InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) = + drop_instance.def + { + // we don't actually need to drop anything + } else { + match ty.kind() { + ty::Dynamic(_, _, ty::Dyn) => { + // IN THIS ARM, WE HAVE: + // ty = *mut (dyn Trait) + // which is: exists<T> ( *mut T, Vtable<T: Trait> ) + // args[0] args[1] + // + // args = ( Data, Vtable ) + // | + // v + // /-------\ + // | ... | + // \-------/ + // + let (ptr, vtable) = drop_place.to_ptr_unsized(); + let ptr = ptr.get_addr(fx); + let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable); + + let is_null = fx.bcx.ins().icmp_imm(IntCC::Equal, drop_fn, 0); + let target_block = fx.get_block(target); + let continued = fx.bcx.create_block(); + fx.bcx.ins().brif(is_null, target_block, &[], continued, &[]); + fx.bcx.switch_to_block(continued); + + // FIXME(eddyb) perhaps move some of this logic into + // `Instance::resolve_drop_in_place`? + let virtual_drop = Instance { + def: ty::InstanceKind::Virtual(drop_instance.def_id(), 0), + args: drop_instance.args, + }; + let fn_abi = + RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty()); + + let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); + let sig = fx.bcx.import_signature(sig); + fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]); + } + ty::Dynamic(_, _, ty::DynStar) => { + // IN THIS ARM, WE HAVE: + // ty = *mut (dyn* Trait) + // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>) + // + // args = [ * ] + // | + // v + // ( Data, Vtable ) + // | + // v + // /-------\ + // | ... | + // \-------/ + // + // + // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING + // + // data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer) + // vtable = (*args[0]).1 // loads the vtable out + // (data, vtable) // an equivalent Rust `*mut dyn Trait` + // + // SO THEN WE CAN USE THE ABOVE CODE. + let (data, vtable) = drop_place.to_cvalue(fx).dyn_star_force_data_on_stack(fx); + let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable); + + let is_null = fx.bcx.ins().icmp_imm(IntCC::Equal, drop_fn, 0); + let target_block = fx.get_block(target); + let continued = fx.bcx.create_block(); + fx.bcx.ins().brif(is_null, target_block, &[], continued, &[]); + fx.bcx.switch_to_block(continued); + + let virtual_drop = Instance { + def: ty::InstanceKind::Virtual(drop_instance.def_id(), 0), + args: drop_instance.args, + }; + let fn_abi = + RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty()); + + let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); + let sig = fx.bcx.import_signature(sig); + fx.bcx.ins().call_indirect(sig, drop_fn, &[data]); + } + _ => { + assert!(!matches!(drop_instance.def, InstanceKind::Virtual(_, _))); + + let fn_abi = + RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(drop_instance, ty::List::empty()); + + let arg_value = drop_place.place_ref( + fx, + fx.layout_of(Ty::new_mut_ref(fx.tcx, fx.tcx.lifetimes.re_erased, ty)), + ); + let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0], true); + + let mut call_args: Vec<Value> = arg_value.into_iter().collect::<Vec<_>>(); + + if drop_instance.def.requires_caller_location(fx.tcx) { + // Pass the caller location for `#[track_caller]`. + let caller_location = fx.get_caller_location(source_info); + call_args.extend( + adjust_arg_for_abi(fx, caller_location, &fn_abi.args[1], false).into_iter(), + ); + } + + let func_ref = fx.get_function_ref(drop_instance); + fx.bcx.ins().call(func_ref, &call_args); + } + } + } + + let target_block = fx.get_block(target); + fx.bcx.ins().jump(target_block, &[]); +} diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs new file mode 100644 index 00000000000..06522670029 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs @@ -0,0 +1,303 @@ +//! Argument passing + +use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose}; +use rustc_target::abi::call::{ + ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, Reg, RegKind, +}; +use smallvec::{smallvec, SmallVec}; + +use crate::prelude::*; +use crate::value_and_place::assert_assignable; + +pub(super) trait ArgAbiExt<'tcx> { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]>; + fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>); +} + +fn reg_to_abi_param(reg: Reg) -> AbiParam { + let clif_ty = match (reg.kind, reg.size.bytes()) { + (RegKind::Integer, 1) => types::I8, + (RegKind::Integer, 2) => types::I16, + (RegKind::Integer, 3..=4) => types::I32, + (RegKind::Integer, 5..=8) => types::I64, + (RegKind::Integer, 9..=16) => types::I128, + (RegKind::Float, 4) => types::F32, + (RegKind::Float, 8) => types::F64, + (RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(), + _ => unreachable!("{:?}", reg), + }; + AbiParam::new(clif_ty) +} + +fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -> AbiParam { + match arg_attrs.arg_ext { + RustcArgExtension::None => {} + RustcArgExtension::Zext => param.extension = ArgumentExtension::Uext, + RustcArgExtension::Sext => param.extension = ArgumentExtension::Sext, + } + param +} + +fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> { + let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 { + (0, 0) + } else { + ( + cast.rest.total.bytes() / cast.rest.unit.size.bytes(), + cast.rest.total.bytes() % cast.rest.unit.size.bytes(), + ) + }; + + // Note: Unlike the LLVM equivalent of this code we don't have separate branches for when there + // is no prefix as a single unit, an array and a heterogeneous struct are not represented using + // different types in Cranelift IR. Instead a single array of primitive types is used. + + // Create list of fields in the main structure + let mut args = cast + .prefix + .iter() + .flatten() + .map(|®| reg_to_abi_param(reg)) + .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit))) + .collect::<SmallVec<_>>(); + + // Append final integer + if rem_bytes != 0 { + // Only integers can be really split further. + assert_eq!(cast.rest.unit.kind, RegKind::Integer); + args.push(reg_to_abi_param(Reg { + kind: RegKind::Integer, + size: Size::from_bytes(rem_bytes), + })); + } + + args +} + +impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> { + match self.mode { + PassMode::Ignore => smallvec![], + PassMode::Direct(attrs) => match self.layout.abi { + Abi::Scalar(scalar) => smallvec![apply_arg_attrs_to_abi_param( + AbiParam::new(scalar_to_clif_type(tcx, scalar)), + attrs + )], + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout); + smallvec![AbiParam::new(vector_ty)] + } + _ => unreachable!("{:?}", self.layout.abi), + }, + PassMode::Pair(attrs_a, attrs_b) => match self.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a); + let b = scalar_to_clif_type(tcx, b); + smallvec![ + apply_arg_attrs_to_abi_param(AbiParam::new(a), attrs_a), + apply_arg_attrs_to_abi_param(AbiParam::new(b), attrs_b), + ] + } + _ => unreachable!("{:?}", self.layout.abi), + }, + PassMode::Cast { ref cast, pad_i32 } => { + assert!(!pad_i32, "padding support not yet implemented"); + cast_target_to_abi_params(cast) + } + PassMode::Indirect { attrs, meta_attrs: None, on_stack } => { + if on_stack { + // Abi requires aligning struct size to pointer size + let size = self.layout.size.align_to(tcx.data_layout.pointer_align.abi); + let size = u32::try_from(size.bytes()).unwrap(); + smallvec![apply_arg_attrs_to_abi_param( + AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructArgument(size),), + attrs + )] + } else { + smallvec![apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)] + } + } + PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { + assert!(!on_stack); + smallvec![ + apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs), + apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), meta_attrs), + ] + } + } + } + + fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>) { + match self.mode { + PassMode::Ignore => (None, vec![]), + PassMode::Direct(_) => match self.layout.abi { + Abi::Scalar(scalar) => { + (None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar))]) + } + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout); + (None, vec![AbiParam::new(vector_ty)]) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + PassMode::Pair(_, _) => match self.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a); + let b = scalar_to_clif_type(tcx, b); + (None, vec![AbiParam::new(a), AbiParam::new(b)]) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + PassMode::Cast { ref cast, .. } => { + (None, cast_target_to_abi_params(cast).into_iter().collect()) + } + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack } => { + assert!(!on_stack); + (Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![]) + } + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + unreachable!("unsized return value") + } + } + } +} + +pub(super) fn to_casted_value<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + arg: CValue<'tcx>, + cast: &CastTarget, +) -> SmallVec<[Value; 2]> { + let (ptr, meta) = arg.force_stack(fx); + assert!(meta.is_none()); + let mut offset = 0; + cast_target_to_abi_params(cast) + .into_iter() + .map(|param| { + let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new()); + offset += i64::from(param.value_type.bytes()); + val + }) + .collect() +} + +pub(super) fn from_casted_value<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + block_params: &[Value], + layout: TyAndLayout<'tcx>, + cast: &CastTarget, +) -> CValue<'tcx> { + let abi_params = cast_target_to_abi_params(cast); + let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum(); + let layout_size = u32::try_from(layout.size.bytes()).unwrap(); + let ptr = fx.create_stack_slot( + // Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`. + // It may also be smaller for example when the type is a wrapper around an integer with a + // larger alignment than the integer. + std::cmp::max(abi_param_size, layout_size), + u32::try_from(layout.align.pref.bytes()).unwrap(), + ); + let mut offset = 0; + let mut block_params_iter = block_params.iter().copied(); + for param in abi_params { + let val = ptr.offset_i64(fx, offset).store( + fx, + block_params_iter.next().unwrap(), + MemFlags::new(), + ); + offset += i64::from(param.value_type.bytes()); + val + } + assert_eq!(block_params_iter.next(), None, "Leftover block param"); + CValue::by_ref(ptr, layout) +} + +/// Get a set of values to be passed as function arguments. +pub(super) fn adjust_arg_for_abi<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + arg: CValue<'tcx>, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + is_owned: bool, +) -> SmallVec<[Value; 2]> { + assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty, 16); + match arg_abi.mode { + PassMode::Ignore => smallvec![], + PassMode::Direct(_) => smallvec![arg.load_scalar(fx)], + PassMode::Pair(_, _) => { + let (a, b) = arg.load_scalar_pair(fx); + smallvec![a, b] + } + PassMode::Cast { ref cast, .. } => to_casted_value(fx, arg, cast), + PassMode::Indirect { .. } => { + if is_owned { + match arg.force_stack(fx) { + (ptr, None) => smallvec![ptr.get_addr(fx)], + (ptr, Some(meta)) => smallvec![ptr.get_addr(fx), meta], + } + } else { + // Ownership of the value at the backing storage for an argument is passed to the + // callee per the ABI, so we must make a copy of the argument unless the argument + // local is moved. + let place = CPlace::new_stack_slot(fx, arg.layout()); + place.write_cvalue(fx, arg); + smallvec![place.to_ptr().get_addr(fx)] + } + } + } +} + +/// Create a [`CValue`] containing the value of a function parameter adding clif function parameters +/// as necessary. +pub(super) fn cvalue_for_param<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + local: Option<mir::Local>, + local_field: Option<usize>, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + block_params_iter: &mut impl Iterator<Item = Value>, +) -> Option<CValue<'tcx>> { + let block_params = arg_abi + .get_abi_param(fx.tcx) + .into_iter() + .map(|abi_param| { + let block_param = block_params_iter.next().unwrap(); + assert_eq!(fx.bcx.func.dfg.value_type(block_param), abi_param.value_type); + block_param + }) + .collect::<SmallVec<[_; 2]>>(); + + crate::abi::comments::add_arg_comment( + fx, + "arg", + local, + local_field, + &block_params, + &arg_abi.mode, + arg_abi.layout, + ); + + match arg_abi.mode { + PassMode::Ignore => None, + PassMode::Direct(_) => { + assert_eq!(block_params.len(), 1, "{:?}", block_params); + Some(CValue::by_val(block_params[0], arg_abi.layout)) + } + PassMode::Pair(_, _) => { + assert_eq!(block_params.len(), 2, "{:?}", block_params); + Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout)) + } + PassMode::Cast { ref cast, .. } => { + Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) + } + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + assert_eq!(block_params.len(), 1, "{:?}", block_params); + Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) + } + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + assert_eq!(block_params.len(), 2, "{:?}", block_params); + Some(CValue::by_ref_unsized( + Pointer::new(block_params[0]), + block_params[1], + arg_abi.layout, + )) + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs new file mode 100644 index 00000000000..e0f399e616e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs @@ -0,0 +1,130 @@ +//! Return value handling + +use rustc_target::abi::call::{ArgAbi, PassMode}; +use smallvec::{smallvec, SmallVec}; + +use crate::prelude::*; + +/// Return a place where the return value of the current function can be written to. If necessary +/// this adds an extra parameter pointing to where the return value needs to be stored. +pub(super) fn codegen_return_param<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + ssa_analyzed: &rustc_index::IndexSlice<Local, crate::analyze::SsaKind>, + block_params_iter: &mut impl Iterator<Item = Value>, +) -> CPlace<'tcx> { + let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.ret.mode { + PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => { + let is_ssa = ssa_analyzed[RETURN_PLACE].is_ssa(fx, fx.fn_abi.ret.layout.ty); + (super::make_local_place(fx, RETURN_PLACE, fx.fn_abi.ret.layout, is_ssa), smallvec![]) + } + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + let ret_param = block_params_iter.next().unwrap(); + assert_eq!(fx.bcx.func.dfg.value_type(ret_param), fx.pointer_type); + (CPlace::for_ptr(Pointer::new(ret_param), fx.fn_abi.ret.layout), smallvec![ret_param]) + } + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + unreachable!("unsized return value") + } + }; + + crate::abi::comments::add_arg_comment( + fx, + "ret", + Some(RETURN_PLACE), + None, + &ret_param, + &fx.fn_abi.ret.mode, + fx.fn_abi.ret.layout, + ); + + ret_place +} + +/// Invokes the closure with if necessary a value representing the return pointer. When the closure +/// returns the call return value(s) if any are written to the correct place. +pub(super) fn codegen_with_call_return_arg<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + ret_place: CPlace<'tcx>, + f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option<Value>) -> Inst, +) { + let (ret_temp_place, return_ptr) = match ret_arg_abi.mode { + PassMode::Ignore => (None, None), + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + if let Some(ret_ptr) = ret_place.try_to_ptr() { + // This is an optimization to prevent unnecessary copies of the return value when + // the return place is already a memory place as opposed to a register. + // This match arm can be safely removed. + (None, Some(ret_ptr.get_addr(fx))) + } else { + let place = CPlace::new_stack_slot(fx, ret_arg_abi.layout); + (Some(place), Some(place.to_ptr().get_addr(fx))) + } + } + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + unreachable!("unsized return value") + } + PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => (None, None), + }; + + let call_inst = f(fx, return_ptr); + + match ret_arg_abi.mode { + PassMode::Ignore => {} + PassMode::Direct(_) => { + let ret_val = fx.bcx.inst_results(call_inst)[0]; + ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_arg_abi.layout)); + } + PassMode::Pair(_, _) => { + let ret_val_a = fx.bcx.inst_results(call_inst)[0]; + let ret_val_b = fx.bcx.inst_results(call_inst)[1]; + ret_place + .write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout)); + } + PassMode::Cast { ref cast, .. } => { + let results = + fx.bcx.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>(); + let result = + super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast); + ret_place.write_cvalue(fx, result); + } + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + if let Some(ret_temp_place) = ret_temp_place { + // If ret_temp_place is None, it is not necessary to copy the return value. + let ret_temp_value = ret_temp_place.to_cvalue(fx); + ret_place.write_cvalue(fx, ret_temp_value); + } + } + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + unreachable!("unsized return value") + } + } +} + +/// Codegen a return instruction with the right return value(s) if any. +pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) { + match fx.fn_abi.ret.mode { + PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + fx.bcx.ins().return_(&[]); + } + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + unreachable!("unsized return value") + } + PassMode::Direct(_) => { + let place = fx.get_local_place(RETURN_PLACE); + let ret_val = place.to_cvalue(fx).load_scalar(fx); + fx.bcx.ins().return_(&[ret_val]); + } + PassMode::Pair(_, _) => { + let place = fx.get_local_place(RETURN_PLACE); + let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); + fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); + } + PassMode::Cast { ref cast, .. } => { + let place = fx.get_local_place(RETURN_PLACE); + let ret_val = place.to_cvalue(fx); + let ret_vals = super::pass_mode::to_casted_value(fx, ret_val, cast); + fx.bcx.ins().return_(&ret_vals); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs new file mode 100644 index 00000000000..b4a3825e996 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/allocator.rs @@ -0,0 +1,96 @@ +//! Allocator shim +// Adapted from rustc + +use rustc_ast::expand::allocator::{ + alloc_error_handler_name, default_fn_name, global_fn_name, AllocatorKind, AllocatorTy, + ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE, +}; +use rustc_codegen_ssa::base::allocator_kind_for_codegen; +use rustc_session::config::OomStrategy; + +use crate::prelude::*; + +/// Returns whether an allocator shim was created +pub(crate) fn codegen(tcx: TyCtxt<'_>, module: &mut dyn Module) -> bool { + let Some(kind) = allocator_kind_for_codegen(tcx) else { return false }; + codegen_inner( + module, + kind, + tcx.alloc_error_handler_kind(()).unwrap(), + tcx.sess.opts.unstable_opts.oom, + ); + true +} + +fn codegen_inner( + module: &mut dyn Module, + kind: AllocatorKind, + alloc_error_handler_kind: AllocatorKind, + oom_strategy: OomStrategy, +) { + let usize_ty = module.target_config().pointer_type(); + + if kind == AllocatorKind::Default { + for method in ALLOCATOR_METHODS { + let mut arg_tys = Vec::with_capacity(method.inputs.len()); + for input in method.inputs.iter() { + match input.ty { + AllocatorTy::Layout => { + arg_tys.push(usize_ty); // size + arg_tys.push(usize_ty); // align + } + AllocatorTy::Ptr => arg_tys.push(usize_ty), + AllocatorTy::Usize => arg_tys.push(usize_ty), + + AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + } + } + let output = match method.output { + AllocatorTy::ResultPtr => Some(usize_ty), + AllocatorTy::Unit => None, + + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + + let sig = Signature { + call_conv: module.target_config().default_call_conv, + params: arg_tys.iter().cloned().map(AbiParam::new).collect(), + returns: output.into_iter().map(AbiParam::new).collect(), + }; + crate::common::create_wrapper_function( + module, + sig, + &global_fn_name(method.name), + &default_fn_name(method.name), + ); + } + } + + let sig = Signature { + call_conv: module.target_config().default_call_conv, + params: vec![AbiParam::new(usize_ty), AbiParam::new(usize_ty)], + returns: vec![], + }; + crate::common::create_wrapper_function( + module, + sig, + "__rust_alloc_error_handler", + &alloc_error_handler_name(alloc_error_handler_kind), + ); + + let data_id = module.declare_data(OomStrategy::SYMBOL, Linkage::Export, false, false).unwrap(); + let mut data = DataDescription::new(); + data.set_align(1); + let val = oom_strategy.should_panic(); + data.define(Box::new([val])); + module.define_data(data_id, &data).unwrap(); + + let data_id = + module.declare_data(NO_ALLOC_SHIM_IS_UNSTABLE, Linkage::Export, false, false).unwrap(); + let mut data = DataDescription::new(); + data.set_align(1); + data.define(Box::new([0])); + module.define_data(data_id, &data).unwrap(); +} diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs new file mode 100644 index 00000000000..c5762638a6b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/analyze.rs @@ -0,0 +1,39 @@ +//! SSA analysis + +use rustc_index::IndexVec; +use rustc_middle::mir::StatementKind::*; + +use crate::prelude::*; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) enum SsaKind { + NotSsa, + MaybeSsa, +} + +impl SsaKind { + pub(crate) fn is_ssa<'tcx>(self, fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { + self == SsaKind::MaybeSsa && (fx.clif_type(ty).is_some() || fx.clif_pair_type(ty).is_some()) + } +} + +pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> { + let mut flag_map = + fx.mir.local_decls.iter().map(|_| SsaKind::MaybeSsa).collect::<IndexVec<Local, SsaKind>>(); + + for bb in fx.mir.basic_blocks.iter() { + for stmt in bb.statements.iter() { + match &stmt.kind { + Assign(place_and_rval) => match &place_and_rval.1 { + Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => { + flag_map[place.local] = SsaKind::NotSsa; + } + _ => {} + }, + _ => {} + } + } + } + + flag_map +} diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs new file mode 100644 index 00000000000..5eedab4f2cb --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/archive.rs @@ -0,0 +1,93 @@ +use std::borrow::Borrow; +use std::fs; +use std::path::Path; + +use ar_archive_writer::{COFFShortExport, MachineTypes}; +use rustc_codegen_ssa::back::archive::{ + create_mingw_dll_import_lib, ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder, + DEFAULT_OBJECT_READER, +}; +use rustc_codegen_ssa::common::is_mingw_gnu_toolchain; +use rustc_session::Session; + +pub(crate) struct ArArchiveBuilderBuilder; + +impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder { + fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> { + Box::new(ArArchiveBuilder::new(sess, &DEFAULT_OBJECT_READER)) + } + + fn create_dll_import_lib( + &self, + sess: &Session, + lib_name: &str, + import_name_and_ordinal_vector: Vec<(String, Option<u16>)>, + output_path: &Path, + ) { + if is_mingw_gnu_toolchain(&sess.target) { + // The binutils linker used on -windows-gnu targets cannot read the import + // libraries generated by LLVM: in our attempts, the linker produced an .EXE + // that loaded but crashed with an AV upon calling one of the imported + // functions. Therefore, use binutils to create the import library instead, + // by writing a .DEF file to the temp dir and calling binutils's dlltool. + create_mingw_dll_import_lib( + sess, + lib_name, + import_name_and_ordinal_vector, + output_path, + ); + } else { + let mut file = + match fs::OpenOptions::new().write(true).create_new(true).open(&output_path) { + Ok(file) => file, + Err(error) => { + sess.dcx().fatal(format!( + "failed to create import library file `{path}`: {error}", + path = output_path.display(), + )); + } + }; + + let machine = match sess.target.arch.borrow() { + "x86" => MachineTypes::I386, + "x86_64" => MachineTypes::AMD64, + "arm" => MachineTypes::ARMNT, + "aarch64" => MachineTypes::ARM64, + _ => { + sess.dcx().fatal(format!( + "unsupported target architecture `{arch}`", + arch = sess.target.arch, + )); + } + }; + + let exports = import_name_and_ordinal_vector + .iter() + .map(|(name, ordinal)| COFFShortExport { + name: name.to_string(), + ext_name: None, + symbol_name: None, + alias_target: None, + ordinal: ordinal.unwrap_or(0), + noname: ordinal.is_some(), + data: false, + private: false, + constant: false, + }) + .collect::<Vec<_>>(); + + if let Err(error) = ar_archive_writer::write_import_library( + &mut file, + lib_name, + &exports, + machine, + !sess.target.is_like_msvc, + ) { + sess.dcx().fatal(format!( + "failed to create import library `{path}`: `{error}`", + path = output_path.display(), + )); + } + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs new file mode 100644 index 00000000000..9bc7b57c537 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -0,0 +1,1083 @@ +//! Codegen of a single function + +use cranelift_codegen::ir::UserFuncName; +use cranelift_codegen::CodegenError; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use cranelift_module::ModuleError; +use rustc_ast::InlineAsmOptions; +use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; +use rustc_index::IndexVec; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::ty::adjustment::PointerCoercion; +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::ty::TypeVisitableExt; + +use crate::constant::ConstantCx; +use crate::debuginfo::{FunctionDebugContext, TypeDebugContext}; +use crate::inline_asm::codegen_naked_asm; +use crate::prelude::*; +use crate::pretty_clif::CommentWriter; + +pub(crate) struct CodegenedFunction { + symbol_name: String, + func_id: FuncId, + func: Function, + clif_comments: CommentWriter, + func_debug_cx: Option<FunctionDebugContext>, +} + +pub(crate) fn codegen_fn<'tcx>( + tcx: TyCtxt<'tcx>, + cx: &mut crate::CodegenCx, + type_dbg: &mut TypeDebugContext<'tcx>, + cached_func: Function, + module: &mut dyn Module, + instance: Instance<'tcx>, +) -> Option<CodegenedFunction> { + debug_assert!(!instance.args.has_infer()); + + let symbol_name = tcx.symbol_name(instance).name.to_string(); + let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name); + + let mir = tcx.instance_mir(instance.def); + let _mir_guard = crate::PrintOnPanic(|| { + let mut buf = Vec::new(); + with_no_trimmed_paths!({ + rustc_middle::mir::pretty::write_mir_fn(tcx, mir, &mut |_, _| Ok(()), &mut buf) + .unwrap(); + }); + String::from_utf8_lossy(&buf).into_owned() + }); + + if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) { + assert_eq!(mir.basic_blocks.len(), 1); + assert!(mir.basic_blocks[START_BLOCK].statements.is_empty()); + + match &mir.basic_blocks[START_BLOCK].terminator().kind { + TerminatorKind::InlineAsm { + template, + operands, + options, + line_spans: _, + targets: _, + unwind: _, + } => { + codegen_naked_asm( + tcx, + cx, + module, + instance, + mir.basic_blocks[START_BLOCK].terminator().source_info.span, + &symbol_name, + template, + operands, + *options, + ); + } + _ => unreachable!(), + } + + return None; + } + + // Declare function + let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance); + let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap(); + + // Make the FunctionBuilder + let mut func_ctx = FunctionBuilderContext::new(); + let mut func = cached_func; + func.clear(); + func.name = UserFuncName::user(0, func_id.as_u32()); + func.signature = sig; + func.collect_debug_info(); + + let mut bcx = FunctionBuilder::new(&mut func, &mut func_ctx); + + // Predefine blocks + let start_block = bcx.create_block(); + let block_map: IndexVec<BasicBlock, Block> = + (0..mir.basic_blocks.len()).map(|_| bcx.create_block()).collect(); + + // Make FunctionCx + let target_config = module.target_config(); + let pointer_type = target_config.pointer_type(); + let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); + + let fn_abi = RevealAllLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty()); + + let func_debug_cx = if let Some(debug_context) = &mut cx.debug_context { + Some(debug_context.define_function(tcx, type_dbg, instance, fn_abi, &symbol_name, mir.span)) + } else { + None + }; + + let mut fx = FunctionCx { + cx, + module, + tcx, + target_config, + pointer_type, + constants_cx: ConstantCx::new(), + func_debug_cx, + + instance, + symbol_name, + mir, + fn_abi, + + bcx, + block_map, + local_map: IndexVec::with_capacity(mir.local_decls.len()), + caller_location: None, // set by `codegen_fn_prelude` + + clif_comments, + next_ssa_var: 0, + }; + + tcx.prof.generic_activity("codegen clif ir").run(|| codegen_fn_body(&mut fx, start_block)); + fx.bcx.seal_all_blocks(); + fx.bcx.finalize(); + + // Recover all necessary data from fx, before accessing func will prevent future access to it. + let symbol_name = fx.symbol_name; + let clif_comments = fx.clif_comments; + let func_debug_cx = fx.func_debug_cx; + + fx.constants_cx.finalize(fx.tcx, &mut *fx.module); + + if cx.should_write_ir { + crate::pretty_clif::write_clif_file( + tcx.output_filenames(()), + &symbol_name, + "unopt", + module.isa(), + &func, + &clif_comments, + ); + } + + // Verify function + verify_func(tcx, &clif_comments, &func); + + Some(CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }) +} + +pub(crate) fn compile_fn( + cx: &mut crate::CodegenCx, + cached_context: &mut Context, + module: &mut dyn Module, + codegened_func: CodegenedFunction, +) { + let _timer = + cx.profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name); + + let clif_comments = codegened_func.clif_comments; + + // Store function in context + let context = cached_context; + context.clear(); + context.func = codegened_func.func; + + #[cfg(any())] // This is never true + let _clif_guard = { + use std::fmt::Write; + + let func_clone = context.func.clone(); + let clif_comments_clone = clif_comments.clone(); + let mut clif = String::new(); + for flag in module.isa().flags().iter() { + writeln!(clif, "set {}", flag).unwrap(); + } + write!(clif, "target {}", module.isa().triple().architecture.to_string()).unwrap(); + for isa_flag in module.isa().isa_flags().iter() { + write!(clif, " {}", isa_flag).unwrap(); + } + writeln!(clif, "\n").unwrap(); + writeln!(clif, "; symbol {}", codegened_func.symbol_name).unwrap(); + crate::PrintOnPanic(move || { + let mut clif = clif.clone(); + ::cranelift_codegen::write::decorate_function( + &mut &clif_comments_clone, + &mut clif, + &func_clone, + ) + .unwrap(); + clif + }) + }; + + // Define function + cx.profiler.generic_activity("define function").run(|| { + context.want_disasm = cx.should_write_ir; + match module.define_function(codegened_func.func_id, context) { + Ok(()) => {} + Err(ModuleError::Compilation(CodegenError::ImplLimitExceeded)) => { + let early_dcx = rustc_session::EarlyDiagCtxt::new( + rustc_session::config::ErrorOutputType::default(), + ); + early_dcx.early_fatal(format!( + "backend implementation limit exceeded while compiling {name}", + name = codegened_func.symbol_name + )); + } + Err(err) => { + panic!("Error while defining {name}: {err:?}", name = codegened_func.symbol_name); + } + } + }); + + if cx.should_write_ir { + // Write optimized function to file for debugging + crate::pretty_clif::write_clif_file( + &cx.output_filenames, + &codegened_func.symbol_name, + "opt", + module.isa(), + &context.func, + &clif_comments, + ); + + if let Some(disasm) = &context.compiled_code().unwrap().vcode { + crate::pretty_clif::write_ir_file( + &cx.output_filenames, + &format!("{}.vcode", codegened_func.symbol_name), + |file| file.write_all(disasm.as_bytes()), + ) + } + } + + // Define debuginfo for function + let debug_context = &mut cx.debug_context; + cx.profiler.generic_activity("generate debug info").run(|| { + if let Some(debug_context) = debug_context { + codegened_func.func_debug_cx.unwrap().finalize( + debug_context, + codegened_func.func_id, + context, + ); + } + }); +} + +pub(crate) fn verify_func( + tcx: TyCtxt<'_>, + writer: &crate::pretty_clif::CommentWriter, + func: &Function, +) { + tcx.prof.generic_activity("verify clif ir").run(|| { + let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder()); + match cranelift_codegen::verify_function(&func, &flags) { + Ok(_) => {} + Err(err) => { + tcx.dcx().err(format!("{:?}", err)); + let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error( + &func, + Some(Box::new(writer)), + err, + ); + tcx.dcx().fatal(format!("cranelift verify error:\n{}", pretty_error)); + } + } + }); +} + +fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { + let arg_uninhabited = fx + .mir + .args_iter() + .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); + if arg_uninhabited { + fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); + fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + return; + } + fx.tcx + .prof + .generic_activity("codegen prelude") + .run(|| crate::abi::codegen_fn_prelude(fx, start_block)); + + let reachable_blocks = traversal::mono_reachable_as_bitset(fx.mir, fx.tcx, fx.instance); + + for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() { + let block = fx.get_block(bb); + fx.bcx.switch_to_block(block); + + if !reachable_blocks.contains(bb) { + // We want to skip this block, because it's not reachable. But we still create + // the block so terminators in other blocks can reference it. + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + continue; + } + + if bb_data.is_cleanup { + // Unwinding after panicking is not supported + continue; + + // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do + // so for cleanup blocks. + } + + fx.bcx.ins().nop(); + for stmt in &bb_data.statements { + fx.set_debug_loc(stmt.source_info); + codegen_stmt(fx, block, stmt); + } + + if fx.clif_comments.enabled() { + let mut terminator_head = "\n".to_string(); + with_no_trimmed_paths!({ + bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap(); + }); + let inst = fx.bcx.func.layout.last_inst(block).unwrap(); + fx.add_comment(inst, terminator_head); + } + + let source_info = bb_data.terminator().source_info; + fx.set_debug_loc(source_info); + + let _print_guard = + crate::PrintOnPanic(|| format!("terminator {:?}", bb_data.terminator().kind)); + + match &bb_data.terminator().kind { + TerminatorKind::Goto { target } => { + if let TerminatorKind::Return = fx.mir[*target].terminator().kind { + let mut can_immediately_return = true; + for stmt in &fx.mir[*target].statements { + if let StatementKind::StorageDead(_) = stmt.kind { + } else { + // FIXME Can sometimes happen, see rust-lang/rust#70531 + can_immediately_return = false; + break; + } + } + + if can_immediately_return { + crate::abi::codegen_return(fx); + continue; + } + } + + let block = fx.get_block(*target); + fx.bcx.ins().jump(block, &[]); + } + TerminatorKind::Return => { + crate::abi::codegen_return(fx); + } + TerminatorKind::Assert { cond, expected, msg, target, unwind: _ } => { + if !fx.tcx.sess.overflow_checks() && msg.is_optional_overflow_check() { + let target = fx.get_block(*target); + fx.bcx.ins().jump(target, &[]); + continue; + } + let cond = codegen_operand(fx, cond).load_scalar(fx); + + let target = fx.get_block(*target); + let failure = fx.bcx.create_block(); + fx.bcx.set_cold_block(failure); + + if *expected { + fx.bcx.ins().brif(cond, target, &[], failure, &[]); + } else { + fx.bcx.ins().brif(cond, failure, &[], target, &[]); + }; + + fx.bcx.switch_to_block(failure); + fx.bcx.ins().nop(); + + match &**msg { + AssertKind::BoundsCheck { ref len, ref index } => { + let len = codegen_operand(fx, len).load_scalar(fx); + let index = codegen_operand(fx, index).load_scalar(fx); + let location = fx.get_caller_location(source_info).load_scalar(fx); + + codegen_panic_inner( + fx, + rustc_hir::LangItem::PanicBoundsCheck, + &[index, len, location], + Some(source_info.span), + ); + } + AssertKind::MisalignedPointerDereference { ref required, ref found } => { + let required = codegen_operand(fx, required).load_scalar(fx); + let found = codegen_operand(fx, found).load_scalar(fx); + let location = fx.get_caller_location(source_info).load_scalar(fx); + + codegen_panic_inner( + fx, + rustc_hir::LangItem::PanicMisalignedPointerDereference, + &[required, found, location], + Some(source_info.span), + ); + } + _ => { + let location = fx.get_caller_location(source_info).load_scalar(fx); + + codegen_panic_inner( + fx, + msg.panic_function(), + &[location], + Some(source_info.span), + ); + } + } + } + + TerminatorKind::SwitchInt { discr, targets } => { + let discr = codegen_operand(fx, discr); + let switch_ty = discr.layout().ty; + let discr = discr.load_scalar(fx); + + let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind() + || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0); + if use_bool_opt { + assert_eq!(targets.iter().count(), 1); + let (then_value, then_block) = targets.iter().next().unwrap(); + let then_block = fx.get_block(then_block); + let else_block = fx.get_block(targets.otherwise()); + let test_zero = match then_value { + 0 => true, + 1 => false, + _ => unreachable!("{:?}", targets), + }; + + let (discr, is_inverted) = + crate::optimize::peephole::maybe_unwrap_bool_not(&mut fx.bcx, discr); + let test_zero = if is_inverted { !test_zero } else { test_zero }; + if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken( + &fx.bcx, discr, test_zero, + ) { + if taken { + fx.bcx.ins().jump(then_block, &[]); + } else { + fx.bcx.ins().jump(else_block, &[]); + } + } else { + if test_zero { + fx.bcx.ins().brif(discr, else_block, &[], then_block, &[]); + } else { + fx.bcx.ins().brif(discr, then_block, &[], else_block, &[]); + } + } + } else { + let mut switch = ::cranelift_frontend::Switch::new(); + for (value, block) in targets.iter() { + let block = fx.get_block(block); + switch.set_entry(value, block); + } + let otherwise_block = fx.get_block(targets.otherwise()); + switch.emit(&mut fx.bcx, discr, otherwise_block); + } + } + TerminatorKind::Call { + func, + args, + destination, + target, + fn_span, + unwind: _, + call_source: _, + } => { + fx.tcx.prof.generic_activity("codegen call").run(|| { + crate::abi::codegen_terminator_call( + fx, + mir::SourceInfo { span: *fn_span, ..source_info }, + func, + args, + *destination, + *target, + ) + }); + } + // FIXME(explicit_tail_calls): add support for tail calls to the cranelift backend, once cranelift supports tail calls + TerminatorKind::TailCall { fn_span, .. } => span_bug!( + *fn_span, + "tail calls are not yet supported in `rustc_codegen_cranelift` backend" + ), + TerminatorKind::InlineAsm { + template, + operands, + options, + targets, + line_spans: _, + unwind: _, + } => { + if options.contains(InlineAsmOptions::MAY_UNWIND) { + fx.tcx.dcx().span_fatal( + source_info.span, + "cranelift doesn't support unwinding from inline assembly.", + ); + } + + let have_labels = if options.contains(InlineAsmOptions::NORETURN) { + !targets.is_empty() + } else { + targets.len() > 1 + }; + if have_labels { + fx.tcx.dcx().span_fatal( + source_info.span, + "cranelift doesn't support labels in inline assembly.", + ); + } + + crate::inline_asm::codegen_inline_asm_terminator( + fx, + source_info.span, + template, + operands, + *options, + targets.get(0).copied(), + ); + } + TerminatorKind::UnwindTerminate(reason) => { + codegen_unwind_terminate(fx, source_info, *reason); + } + TerminatorKind::UnwindResume => { + // FIXME implement unwinding + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + } + TerminatorKind::Unreachable => { + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + } + TerminatorKind::Yield { .. } + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } + | TerminatorKind::CoroutineDrop => { + bug!("shouldn't exist at codegen {:?}", bb_data.terminator()); + } + TerminatorKind::Drop { place, target, unwind: _, replace: _ } => { + let drop_place = codegen_place(fx, *place); + crate::abi::codegen_drop(fx, source_info, drop_place, *target); + } + }; + } +} + +fn codegen_stmt<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + #[allow(unused_variables)] cur_block: Block, + stmt: &Statement<'tcx>, +) { + let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt)); + + fx.set_debug_loc(stmt.source_info); + + match &stmt.kind { + StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful + _ => { + if fx.clif_comments.enabled() { + let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); + with_no_trimmed_paths!({ + fx.add_comment(inst, format!("{:?}", stmt)); + }); + } + } + } + + match &stmt.kind { + StatementKind::SetDiscriminant { place, variant_index } => { + let place = codegen_place(fx, **place); + crate::discriminant::codegen_set_discriminant(fx, place, *variant_index); + } + StatementKind::Assign(to_place_and_rval) => { + let lval = codegen_place(fx, to_place_and_rval.0); + let dest_layout = lval.layout(); + match to_place_and_rval.1 { + Rvalue::Use(ref operand) => { + let val = codegen_operand(fx, operand); + lval.write_cvalue(fx, val); + } + Rvalue::CopyForDeref(place) => { + let cplace = codegen_place(fx, place); + let val = cplace.to_cvalue(fx); + lval.write_cvalue(fx, val) + } + Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => { + let place = codegen_place(fx, place); + let ref_ = place.place_ref(fx, lval.layout()); + lval.write_cvalue(fx, ref_); + } + Rvalue::ThreadLocalRef(def_id) => { + let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout()); + lval.write_cvalue(fx, val); + } + Rvalue::BinaryOp(bin_op, ref lhs_rhs) => { + let lhs = codegen_operand(fx, &lhs_rhs.0); + let rhs = codegen_operand(fx, &lhs_rhs.1); + + let res = if let Some(bin_op) = bin_op.overflowing_to_wrapping() { + crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs) + } else { + crate::num::codegen_binop(fx, bin_op, lhs, rhs) + }; + lval.write_cvalue(fx, res); + } + Rvalue::UnaryOp(un_op, ref operand) => { + let operand = codegen_operand(fx, operand); + let layout = operand.layout(); + let res = match un_op { + UnOp::Not => { + let val = operand.load_scalar(fx); + match layout.ty.kind() { + ty::Bool => { + let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0); + CValue::by_val(res, layout) + } + ty::Uint(_) | ty::Int(_) => { + CValue::by_val(fx.bcx.ins().bnot(val), layout) + } + _ => unreachable!("un op Not for {:?}", layout.ty), + } + } + UnOp::Neg => { + let val = operand.load_scalar(fx); + match layout.ty.kind() { + ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout), + ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout), + _ => unreachable!("un op Neg for {:?}", layout.ty), + } + } + UnOp::PtrMetadata => match layout.abi { + Abi::Scalar(_) => CValue::zst(dest_layout), + Abi::ScalarPair(_, _) => { + CValue::by_val(operand.load_scalar_pair(fx).1, dest_layout) + } + _ => bug!("Unexpected `PtrToMetadata` operand: {operand:?}"), + }, + }; + lval.write_cvalue(fx, res); + } + Rvalue::Cast( + CastKind::PointerCoercion(PointerCoercion::ReifyFnPointer), + ref operand, + to_ty, + ) => { + let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx)); + let to_layout = fx.layout_of(fx.monomorphize(to_ty)); + match *from_ty.kind() { + ty::FnDef(def_id, args) => { + let func_ref = fx.get_function_ref( + Instance::resolve_for_fn_ptr( + fx.tcx, + ParamEnv::reveal_all(), + def_id, + args, + ) + .unwrap() + .polymorphize(fx.tcx), + ); + let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); + lval.write_cvalue(fx, CValue::by_val(func_addr, to_layout)); + } + _ => bug!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty), + } + } + Rvalue::Cast( + CastKind::PointerCoercion(PointerCoercion::UnsafeFnPointer), + ref operand, + to_ty, + ) => { + let to_layout = fx.layout_of(fx.monomorphize(to_ty)); + let operand = codegen_operand(fx, operand); + lval.write_cvalue(fx, operand.cast_pointer_to(to_layout)); + } + Rvalue::Cast( + CastKind::PointerCoercion( + PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer, + ), + .., + ) => { + bug!( + "{:?} is for borrowck, and should never appear in codegen", + to_place_and_rval.1 + ); + } + Rvalue::Cast( + CastKind::IntToInt + | CastKind::FloatToFloat + | CastKind::FloatToInt + | CastKind::IntToFloat + | CastKind::FnPtrToPtr + | CastKind::PtrToPtr + | CastKind::PointerExposeProvenance + | CastKind::PointerWithExposedProvenance, + ref operand, + to_ty, + ) => { + let operand = codegen_operand(fx, operand); + let from_ty = operand.layout().ty; + let to_ty = fx.monomorphize(to_ty); + + fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { + ty.builtin_deref(true) + .is_some_and(|pointee_ty| has_ptr_meta(fx.tcx, pointee_ty)) + } + + if is_fat_ptr(fx, from_ty) { + if is_fat_ptr(fx, to_ty) { + // fat-ptr -> fat-ptr + lval.write_cvalue(fx, operand.cast_pointer_to(dest_layout)); + } else { + // fat-ptr -> thin-ptr + let (ptr, _extra) = operand.load_scalar_pair(fx); + lval.write_cvalue(fx, CValue::by_val(ptr, dest_layout)) + } + } else { + let to_clif_ty = fx.clif_type(to_ty).unwrap(); + let from = operand.load_scalar(fx); + + let res = clif_int_or_float_cast( + fx, + from, + type_sign(from_ty), + to_clif_ty, + type_sign(to_ty), + ); + lval.write_cvalue(fx, CValue::by_val(res, dest_layout)); + } + } + Rvalue::Cast( + CastKind::PointerCoercion(PointerCoercion::ClosureFnPointer(_)), + ref operand, + _to_ty, + ) => { + let operand = codegen_operand(fx, operand); + match *operand.layout().ty.kind() { + ty::Closure(def_id, args) => { + let instance = Instance::resolve_closure( + fx.tcx, + def_id, + args, + ty::ClosureKind::FnOnce, + ) + .polymorphize(fx.tcx); + let func_ref = fx.get_function_ref(instance); + let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); + lval.write_cvalue(fx, CValue::by_val(func_addr, lval.layout())); + } + _ => bug!("{} cannot be cast to a fn ptr", operand.layout().ty), + } + } + Rvalue::Cast( + CastKind::PointerCoercion(PointerCoercion::Unsize), + ref operand, + _to_ty, + ) => { + let operand = codegen_operand(fx, operand); + crate::unsize::coerce_unsized_into(fx, operand, lval); + } + Rvalue::Cast(CastKind::DynStar, ref operand, _) => { + let operand = codegen_operand(fx, operand); + crate::unsize::coerce_dyn_star(fx, operand, lval); + } + Rvalue::Cast(CastKind::Transmute, ref operand, _to_ty) => { + let operand = codegen_operand(fx, operand); + lval.write_cvalue_transmute(fx, operand); + } + Rvalue::Discriminant(place) => { + let place = codegen_place(fx, place); + let value = place.to_cvalue(fx); + crate::discriminant::codegen_get_discriminant(fx, lval, value, dest_layout); + } + Rvalue::Repeat(ref operand, times) => { + let operand = codegen_operand(fx, operand); + let times = + fx.monomorphize(times).eval_target_usize(fx.tcx, ParamEnv::reveal_all()); + if operand.layout().size.bytes() == 0 { + // Do nothing for ZST's + } else if fx.clif_type(operand.layout().ty) == Some(types::I8) { + let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64); + // FIXME use emit_small_memset where possible + let addr = lval.to_ptr().get_addr(fx); + let val = operand.load_scalar(fx); + fx.bcx.call_memset(fx.target_config, addr, val, times); + } else { + let loop_block = fx.bcx.create_block(); + let loop_block2 = fx.bcx.create_block(); + let done_block = fx.bcx.create_block(); + let index = fx.bcx.append_block_param(loop_block, fx.pointer_type); + let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); + fx.bcx.ins().jump(loop_block, &[zero]); + + fx.bcx.switch_to_block(loop_block); + let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64); + fx.bcx.ins().brif(done, done_block, &[], loop_block2, &[]); + + fx.bcx.switch_to_block(loop_block2); + let to = lval.place_index(fx, index); + to.write_cvalue(fx, operand); + let index = fx.bcx.ins().iadd_imm(index, 1); + fx.bcx.ins().jump(loop_block, &[index]); + + fx.bcx.switch_to_block(done_block); + fx.bcx.ins().nop(); + } + } + Rvalue::Len(place) => { + let place = codegen_place(fx, place); + let usize_layout = fx.layout_of(fx.tcx.types.usize); + let len = codegen_array_len(fx, place); + lval.write_cvalue(fx, CValue::by_val(len, usize_layout)); + } + Rvalue::ShallowInitBox(ref operand, content_ty) => { + let content_ty = fx.monomorphize(content_ty); + let box_layout = fx.layout_of(Ty::new_box(fx.tcx, content_ty)); + let operand = codegen_operand(fx, operand); + let operand = operand.load_scalar(fx); + lval.write_cvalue(fx, CValue::by_val(operand, box_layout)); + } + Rvalue::NullaryOp(ref null_op, ty) => { + assert!(lval.layout().ty.is_sized(fx.tcx, ParamEnv::reveal_all())); + let layout = fx.layout_of(fx.monomorphize(ty)); + let val = match null_op { + NullOp::SizeOf => layout.size.bytes(), + NullOp::AlignOf => layout.align.abi.bytes(), + NullOp::OffsetOf(fields) => fx + .tcx + .offset_of_subfield(ParamEnv::reveal_all(), layout, fields.iter()) + .bytes(), + NullOp::UbChecks => { + let val = fx.tcx.sess.ub_checks(); + let val = CValue::by_val( + fx.bcx.ins().iconst(types::I8, i64::try_from(val).unwrap()), + fx.layout_of(fx.tcx.types.bool), + ); + lval.write_cvalue(fx, val); + return; + } + }; + let val = CValue::by_val( + fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(val).unwrap()), + fx.layout_of(fx.tcx.types.usize), + ); + lval.write_cvalue(fx, val); + } + Rvalue::Aggregate(ref kind, ref operands) + if matches!(**kind, AggregateKind::RawPtr(..)) => + { + let ty = to_place_and_rval.1.ty(&fx.mir.local_decls, fx.tcx); + let layout = fx.layout_of(fx.monomorphize(ty)); + let [data, meta] = &*operands.raw else { + bug!("RawPtr fields: {operands:?}"); + }; + let data = codegen_operand(fx, data); + let meta = codegen_operand(fx, meta); + assert!(data.layout().ty.is_unsafe_ptr()); + assert!(layout.ty.is_unsafe_ptr()); + let ptr_val = if meta.layout().is_zst() { + data.cast_pointer_to(layout) + } else { + CValue::by_val_pair(data.load_scalar(fx), meta.load_scalar(fx), layout) + }; + lval.write_cvalue(fx, ptr_val); + } + Rvalue::Aggregate(ref kind, ref operands) => { + let (variant_index, variant_dest, active_field_index) = match **kind { + mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => { + let variant_dest = lval.downcast_variant(fx, variant_index); + (variant_index, variant_dest, active_field_index) + } + _ => (FIRST_VARIANT, lval, None), + }; + if active_field_index.is_some() { + assert_eq!(operands.len(), 1); + } + for (i, operand) in operands.iter_enumerated() { + let operand = codegen_operand(fx, operand); + let field_index = active_field_index.unwrap_or(i); + let to = if let mir::AggregateKind::Array(_) = **kind { + let array_index = i64::from(field_index.as_u32()); + let index = fx.bcx.ins().iconst(fx.pointer_type, array_index); + variant_dest.place_index(fx, index) + } else { + variant_dest.place_field(fx, field_index) + }; + to.write_cvalue(fx, operand); + } + crate::discriminant::codegen_set_discriminant(fx, lval, variant_index); + } + } + } + StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Deinit(_) + | StatementKind::ConstEvalCounter + | StatementKind::Nop + | StatementKind::FakeRead(..) + | StatementKind::Retag { .. } + | StatementKind::PlaceMention(..) + | StatementKind::AscribeUserType(..) => {} + + StatementKind::Coverage { .. } => unreachable!(), + StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic { + // We ignore `assume` intrinsics, they are only useful for optimizations + NonDivergingIntrinsic::Assume(_) => {} + NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping { + src, + dst, + count, + }) => { + let dst = codegen_operand(fx, dst); + let pointee = dst + .layout() + .pointee_info_at(fx, rustc_target::abi::Size::ZERO) + .expect("Expected pointer"); + let dst = dst.load_scalar(fx); + let src = codegen_operand(fx, src).load_scalar(fx); + let count = codegen_operand(fx, count).load_scalar(fx); + let elem_size: u64 = pointee.size.bytes(); + let bytes = if elem_size != 1 { + fx.bcx.ins().imul_imm(count, elem_size as i64) + } else { + count + }; + fx.bcx.call_memcpy(fx.target_config, dst, src, bytes); + } + }, + } +} + +fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value { + match *place.layout().ty.kind() { + ty::Array(_elem_ty, len) => { + let len = fx.monomorphize(len).eval_target_usize(fx.tcx, ParamEnv::reveal_all()) as i64; + fx.bcx.ins().iconst(fx.pointer_type, len) + } + ty::Slice(_elem_ty) => place.to_ptr_unsized().1, + _ => bug!("Rvalue::Len({:?})", place), + } +} + +pub(crate) fn codegen_place<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + place: Place<'tcx>, +) -> CPlace<'tcx> { + let mut cplace = fx.get_local_place(place.local); + + for elem in place.projection { + match elem { + PlaceElem::Deref => { + cplace = cplace.place_deref(fx); + } + PlaceElem::OpaqueCast(ty) => bug!("encountered OpaqueCast({ty}) in codegen"), + PlaceElem::Subtype(ty) => cplace = cplace.place_transmute_type(fx, fx.monomorphize(ty)), + PlaceElem::Field(field, _ty) => { + cplace = cplace.place_field(fx, field); + } + PlaceElem::Index(local) => { + let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx); + cplace = cplace.place_index(fx, index); + } + PlaceElem::ConstantIndex { offset, min_length: _, from_end } => { + let offset: u64 = offset; + let index = if !from_end { + fx.bcx.ins().iconst(fx.pointer_type, offset as i64) + } else { + let len = codegen_array_len(fx, cplace); + fx.bcx.ins().iadd_imm(len, -(offset as i64)) + }; + cplace = cplace.place_index(fx, index); + } + PlaceElem::Subslice { from, to, from_end } => { + // These indices are generated by slice patterns. + // slice[from:-to] in Python terms. + + let from: u64 = from; + let to: u64 = to; + + match cplace.layout().ty.kind() { + ty::Array(elem_ty, _len) => { + assert!(!from_end, "array subslices are never `from_end`"); + let elem_layout = fx.layout_of(*elem_ty); + let ptr = cplace.to_ptr(); + cplace = CPlace::for_ptr( + ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), + fx.layout_of(Ty::new_array(fx.tcx, *elem_ty, to - from)), + ); + } + ty::Slice(elem_ty) => { + assert!(from_end, "slice subslices should be `from_end`"); + let elem_layout = fx.layout_of(*elem_ty); + let (ptr, len) = cplace.to_ptr_unsized(); + cplace = CPlace::for_ptr_with_extra( + ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), + fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)), + cplace.layout(), + ); + } + _ => unreachable!(), + } + } + PlaceElem::Downcast(_adt_def, variant) => { + cplace = cplace.downcast_variant(fx, variant); + } + } + } + + cplace +} + +pub(crate) fn codegen_operand<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + operand: &Operand<'tcx>, +) -> CValue<'tcx> { + match operand { + Operand::Move(place) | Operand::Copy(place) => { + let cplace = codegen_place(fx, *place); + cplace.to_cvalue(fx) + } + Operand::Constant(const_) => crate::constant::codegen_constant_operand(fx, const_), + } +} + +pub(crate) fn codegen_panic_nounwind<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + msg_str: &str, + span: Option<Span>, +) { + let msg_ptr = fx.anonymous_str(msg_str); + let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); + let args = [msg_ptr, msg_len]; + + codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, span); +} + +pub(crate) fn codegen_unwind_terminate<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + source_info: mir::SourceInfo, + reason: UnwindTerminateReason, +) { + let args = []; + + codegen_panic_inner(fx, reason.lang_item(), &args, Some(source_info.span)); +} + +fn codegen_panic_inner<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + lang_item: rustc_hir::LangItem, + args: &[Value], + span: Option<Span>, +) { + let def_id = fx.tcx.require_lang_item(lang_item, span); + + let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); + + if is_call_from_compiler_builtins_to_upstream_monomorphization(fx.tcx, instance) { + fx.bcx.ins().trap(TrapCode::User(0)); + return; + } + + let symbol_name = fx.tcx.symbol_name(instance).name; + + fx.lib_call( + symbol_name, + args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(), + vec![], + args, + ); + + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); +} diff --git a/compiler/rustc_codegen_cranelift/src/cast.rs b/compiler/rustc_codegen_cranelift/src/cast.rs new file mode 100644 index 00000000000..0b5cb1547fc --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/cast.rs @@ -0,0 +1,168 @@ +//! Various number casting functions + +use crate::prelude::*; + +pub(crate) fn clif_intcast( + fx: &mut FunctionCx<'_, '_, '_>, + val: Value, + to: Type, + signed: bool, +) -> Value { + let from = fx.bcx.func.dfg.value_type(val); + match (from, to) { + // equal + (_, _) if from == to => val, + + // extend + (_, _) if to.wider_or_equal(from) => { + if signed { + fx.bcx.ins().sextend(to, val) + } else { + fx.bcx.ins().uextend(to, val) + } + } + + // reduce + (_, _) => fx.bcx.ins().ireduce(to, val), + } +} + +pub(crate) fn clif_int_or_float_cast( + fx: &mut FunctionCx<'_, '_, '_>, + from: Value, + from_signed: bool, + to_ty: Type, + to_signed: bool, +) -> Value { + let from_ty = fx.bcx.func.dfg.value_type(from); + + if from_ty.is_int() && to_ty.is_int() { + // int-like -> int-like + clif_intcast( + fx, + from, + to_ty, + // This is correct as either from_signed == to_signed (=> this is trivially correct) + // Or from_clif_ty == to_clif_ty, which means this is a no-op. + from_signed, + ) + } else if from_ty.is_int() && to_ty.is_float() { + if from_ty == types::I128 { + // _______ss__f_ + // __float tisf: i128 -> f32 + // __float tidf: i128 -> f64 + // __floatuntisf: u128 -> f32 + // __floatuntidf: u128 -> f64 + + let name = format!( + "__float{sign}ti{flt}f", + sign = if from_signed { "" } else { "un" }, + flt = match to_ty { + types::F32 => "s", + types::F64 => "d", + _ => unreachable!("{:?}", to_ty), + }, + ); + + return fx.lib_call( + &name, + vec![AbiParam::new(types::I128)], + vec![AbiParam::new(to_ty)], + &[from], + )[0]; + } + + // int-like -> float + if from_signed { + fx.bcx.ins().fcvt_from_sint(to_ty, from) + } else { + fx.bcx.ins().fcvt_from_uint(to_ty, from) + } + } else if from_ty.is_float() && to_ty.is_int() { + let val = if to_ty == types::I128 { + // _____sssf___ + // __fix sfti: f32 -> i128 + // __fix dfti: f64 -> i128 + // __fixunssfti: f32 -> u128 + // __fixunsdfti: f64 -> u128 + + let name = format!( + "__fix{sign}{flt}fti", + sign = if to_signed { "" } else { "uns" }, + flt = match from_ty { + types::F32 => "s", + types::F64 => "d", + _ => unreachable!("{:?}", to_ty), + }, + ); + + if fx.tcx.sess.target.is_like_windows { + let ret = fx.lib_call( + &name, + vec![AbiParam::new(from_ty)], + vec![AbiParam::new(types::I64X2)], + &[from], + )[0]; + // FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128 + let ret_ptr = fx.create_stack_slot(16, 16); + ret_ptr.store(fx, ret, MemFlags::trusted()); + ret_ptr.load(fx, types::I128, MemFlags::trusted()) + } else { + fx.lib_call( + &name, + vec![AbiParam::new(from_ty)], + vec![AbiParam::new(types::I128)], + &[from], + )[0] + } + } else if to_ty == types::I8 || to_ty == types::I16 { + // FIXME implement fcvt_to_*int_sat.i8/i16 + let val = if to_signed { + fx.bcx.ins().fcvt_to_sint_sat(types::I32, from) + } else { + fx.bcx.ins().fcvt_to_uint_sat(types::I32, from) + }; + let (min, max) = match (to_ty, to_signed) { + (types::I8, false) => (0, i64::from(u8::MAX)), + (types::I16, false) => (0, i64::from(u16::MAX)), + (types::I8, true) => (i64::from(i8::MIN as u32), i64::from(i8::MAX as u32)), + (types::I16, true) => (i64::from(i16::MIN as u32), i64::from(i16::MAX as u32)), + _ => unreachable!(), + }; + let min_val = fx.bcx.ins().iconst(types::I32, min); + let max_val = fx.bcx.ins().iconst(types::I32, max); + + let val = if to_signed { + let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, min); + let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, val, max); + let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, val); + fx.bcx.ins().select(has_overflow, max_val, bottom_capped) + } else { + let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, val, max); + fx.bcx.ins().select(has_overflow, max_val, val) + }; + fx.bcx.ins().ireduce(to_ty, val) + } else if to_signed { + fx.bcx.ins().fcvt_to_sint_sat(to_ty, from) + } else { + fx.bcx.ins().fcvt_to_uint_sat(to_ty, from) + }; + + if let Some(false) = fx.tcx.sess.opts.unstable_opts.saturating_float_casts { + return val; + } + + let is_not_nan = fx.bcx.ins().fcmp(FloatCC::Equal, from, from); + let zero = type_zero_value(&mut fx.bcx, to_ty); + fx.bcx.ins().select(is_not_nan, val, zero) + } else if from_ty.is_float() && to_ty.is_float() { + // float -> float + match (from_ty, to_ty) { + (types::F32, types::F64) => fx.bcx.ins().fpromote(types::F64, from), + (types::F64, types::F32) => fx.bcx.ins().fdemote(types::F32, from), + _ => from, + } + } else { + unreachable!("cast value from {:?} to {:?}", from_ty, to_ty); + } +} diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs new file mode 100644 index 00000000000..b6a4769e031 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs @@ -0,0 +1,124 @@ +//! Replaces 128-bit operators with lang item calls where necessary + +use cranelift_codegen::ir::ArgumentPurpose; + +use crate::prelude::*; + +pub(crate) fn maybe_codegen<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + lhs: CValue<'tcx>, + rhs: CValue<'tcx>, +) -> Option<CValue<'tcx>> { + if lhs.layout().ty != fx.tcx.types.u128 + && lhs.layout().ty != fx.tcx.types.i128 + && rhs.layout().ty != fx.tcx.types.u128 + && rhs.layout().ty != fx.tcx.types.i128 + { + return None; + } + + let is_signed = type_sign(lhs.layout().ty); + + match bin_op { + BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => None, + BinOp::Add | BinOp::AddUnchecked | BinOp::Sub | BinOp::SubUnchecked => None, + BinOp::Mul | BinOp::MulUnchecked => None, + BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), + BinOp::Div | BinOp::Rem => { + let name = match (bin_op, is_signed) { + (BinOp::Div, false) => "__udivti3", + (BinOp::Div, true) => "__divti3", + (BinOp::Rem, false) => "__umodti3", + (BinOp::Rem, true) => "__modti3", + _ => unreachable!(), + }; + if fx.tcx.sess.target.is_like_windows { + let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)]; + let ret = fx.lib_call( + name, + vec![AbiParam::new(types::I128), AbiParam::new(types::I128)], + vec![AbiParam::new(types::I64X2)], + &args, + )[0]; + // FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128 + let ret_place = CPlace::new_stack_slot(fx, lhs.layout()); + ret_place.to_ptr().store(fx, ret, MemFlags::trusted()); + Some(ret_place.to_cvalue(fx)) + } else { + let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)]; + let ret_val = fx.lib_call( + name, + vec![AbiParam::new(types::I128), AbiParam::new(types::I128)], + vec![AbiParam::new(types::I128)], + &args, + )[0]; + Some(CValue::by_val(ret_val, lhs.layout())) + } + } + BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne | BinOp::Cmp => None, + BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => None, + BinOp::AddWithOverflow | BinOp::SubWithOverflow | BinOp::MulWithOverflow => unreachable!(), + } +} + +pub(crate) fn maybe_codegen_checked<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + lhs: CValue<'tcx>, + rhs: CValue<'tcx>, +) -> Option<CValue<'tcx>> { + if lhs.layout().ty != fx.tcx.types.u128 + && lhs.layout().ty != fx.tcx.types.i128 + && rhs.layout().ty != fx.tcx.types.u128 + && rhs.layout().ty != fx.tcx.types.i128 + { + return None; + } + + let is_signed = type_sign(lhs.layout().ty); + + match bin_op { + BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => unreachable!(), + BinOp::Add | BinOp::Sub => None, + BinOp::Mul if is_signed => { + let out_ty = Ty::new_tup(fx.tcx, &[lhs.layout().ty, fx.tcx.types.bool]); + let oflow = CPlace::new_stack_slot(fx, fx.layout_of(fx.tcx.types.i32)); + let lhs = lhs.load_scalar(fx); + let rhs = rhs.load_scalar(fx); + let oflow_ptr = oflow.to_ptr().get_addr(fx); + let res = fx.lib_call_unadjusted( + "__muloti4", + vec![ + AbiParam::new(types::I128), + AbiParam::new(types::I128), + AbiParam::new(fx.pointer_type), + ], + vec![AbiParam::new(types::I128)], + &[lhs, rhs, oflow_ptr], + )[0]; + let oflow = oflow.to_cvalue(fx).load_scalar(fx); + let oflow = fx.bcx.ins().ireduce(types::I8, oflow); + Some(CValue::by_val_pair(res, oflow, fx.layout_of(out_ty))) + } + BinOp::Mul => { + let out_ty = Ty::new_tup(fx.tcx, &[lhs.layout().ty, fx.tcx.types.bool]); + let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty)); + let param_types = vec![ + AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn), + AbiParam::new(types::I128), + AbiParam::new(types::I128), + ]; + let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)]; + fx.lib_call("__rust_u128_mulo", param_types, vec![], &args); + Some(out_place.to_cvalue(fx)) + } + BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked => unreachable!(), + BinOp::AddWithOverflow | BinOp::SubWithOverflow | BinOp::MulWithOverflow => unreachable!(), + BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), + BinOp::Div | BinOp::Rem => unreachable!(), + BinOp::Cmp => unreachable!(), + BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => unreachable!(), + BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => unreachable!(), + } +} diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs new file mode 100644 index 00000000000..a1b29a42225 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -0,0 +1,518 @@ +use cranelift_codegen::isa::TargetFrontendConfig; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use rustc_index::IndexVec; +use rustc_middle::ty::layout::{ + self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, +}; +use rustc_middle::ty::TypeFoldable; +use rustc_span::source_map::Spanned; +use rustc_target::abi::call::FnAbi; +use rustc_target::abi::{Float, Integer, Primitive}; +use rustc_target::spec::{HasTargetSpec, Target}; + +use crate::constant::ConstantCx; +use crate::debuginfo::FunctionDebugContext; +use crate::prelude::*; + +pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type { + match tcx.data_layout.pointer_size.bits() { + 16 => types::I16, + 32 => types::I32, + 64 => types::I64, + bits => bug!("ptr_sized_integer: unknown pointer bit size {}", bits), + } +} + +pub(crate) fn scalar_to_clif_type(tcx: TyCtxt<'_>, scalar: Scalar) -> Type { + match scalar.primitive() { + Primitive::Int(int, _sign) => match int { + Integer::I8 => types::I8, + Integer::I16 => types::I16, + Integer::I32 => types::I32, + Integer::I64 => types::I64, + Integer::I128 => types::I128, + }, + Primitive::Float(float) => match float { + Float::F16 => unimplemented!("f16_f128"), + Float::F32 => types::F32, + Float::F64 => types::F64, + Float::F128 => unimplemented!("f16_f128"), + }, + // FIXME(erikdesjardins): handle non-default addrspace ptr sizes + Primitive::Pointer(_) => pointer_ty(tcx), + } +} + +fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Type> { + Some(match ty.kind() { + ty::Bool => types::I8, + ty::Uint(size) => match size { + UintTy::U8 => types::I8, + UintTy::U16 => types::I16, + UintTy::U32 => types::I32, + UintTy::U64 => types::I64, + UintTy::U128 => types::I128, + UintTy::Usize => pointer_ty(tcx), + }, + ty::Int(size) => match size { + IntTy::I8 => types::I8, + IntTy::I16 => types::I16, + IntTy::I32 => types::I32, + IntTy::I64 => types::I64, + IntTy::I128 => types::I128, + IntTy::Isize => pointer_ty(tcx), + }, + ty::Char => types::I32, + ty::Float(size) => match size { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => types::F32, + FloatTy::F64 => types::F64, + FloatTy::F128 => unimplemented!("f16_f128"), + }, + ty::FnPtr(_) => pointer_ty(tcx), + ty::RawPtr(pointee_ty, _) | ty::Ref(_, pointee_ty, _) => { + if has_ptr_meta(tcx, *pointee_ty) { + return None; + } else { + pointer_ty(tcx) + } + } + ty::Param(_) => bug!("ty param {:?}", ty), + _ => return None, + }) +} + +fn clif_pair_type_from_ty<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, +) -> Option<(types::Type, types::Type)> { + Some(match ty.kind() { + ty::Tuple(types) if types.len() == 2 => { + (clif_type_from_ty(tcx, types[0])?, clif_type_from_ty(tcx, types[1])?) + } + ty::RawPtr(pointee_ty, _) | ty::Ref(_, pointee_ty, _) => { + if has_ptr_meta(tcx, *pointee_ty) { + (pointer_ty(tcx), pointer_ty(tcx)) + } else { + return None; + } + } + _ => return None, + }) +} + +/// Is a pointer to this type a fat ptr? +pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool { + if ty.is_sized(tcx, ParamEnv::reveal_all()) { + return false; + } + + let tail = tcx.struct_tail_for_codegen(ty, ParamEnv::reveal_all()); + match tail.kind() { + ty::Foreign(..) => false, + ty::Str | ty::Slice(..) | ty::Dynamic(..) => true, + _ => bug!("unexpected unsized tail: {:?}", tail), + } +} + +pub(crate) fn codegen_icmp_imm( + fx: &mut FunctionCx<'_, '_, '_>, + intcc: IntCC, + lhs: Value, + rhs: i128, +) -> Value { + let lhs_ty = fx.bcx.func.dfg.value_type(lhs); + if lhs_ty == types::I128 { + // FIXME legalize `icmp_imm.i128` in Cranelift + + let (lhs_lsb, lhs_msb) = fx.bcx.ins().isplit(lhs); + let (rhs_lsb, rhs_msb) = (rhs as u128 as u64 as i64, (rhs as u128 >> 64) as u64 as i64); + + match intcc { + IntCC::Equal => { + let lsb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_lsb, rhs_lsb); + let msb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_msb, rhs_msb); + fx.bcx.ins().band(lsb_eq, msb_eq) + } + IntCC::NotEqual => { + let lsb_ne = fx.bcx.ins().icmp_imm(IntCC::NotEqual, lhs_lsb, rhs_lsb); + let msb_ne = fx.bcx.ins().icmp_imm(IntCC::NotEqual, lhs_msb, rhs_msb); + fx.bcx.ins().bor(lsb_ne, msb_ne) + } + _ => { + // if msb_eq { + // lsb_cc + // } else { + // msb_cc + // } + + let msb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_msb, rhs_msb); + let lsb_cc = fx.bcx.ins().icmp_imm(intcc, lhs_lsb, rhs_lsb); + let msb_cc = fx.bcx.ins().icmp_imm(intcc, lhs_msb, rhs_msb); + + fx.bcx.ins().select(msb_eq, lsb_cc, msb_cc) + } + } + } else { + let rhs = rhs as i64; // Truncates on purpose in case rhs is actually an unsigned value + fx.bcx.ins().icmp_imm(intcc, lhs, rhs) + } +} + +pub(crate) fn codegen_bitcast(fx: &mut FunctionCx<'_, '_, '_>, dst_ty: Type, val: Value) -> Value { + let mut flags = MemFlags::new(); + flags.set_endianness(match fx.tcx.data_layout.endian { + rustc_target::abi::Endian::Big => cranelift_codegen::ir::Endianness::Big, + rustc_target::abi::Endian::Little => cranelift_codegen::ir::Endianness::Little, + }); + fx.bcx.ins().bitcast(dst_ty, flags, val) +} + +pub(crate) fn type_zero_value(bcx: &mut FunctionBuilder<'_>, ty: Type) -> Value { + if ty == types::I128 { + let zero = bcx.ins().iconst(types::I64, 0); + bcx.ins().iconcat(zero, zero) + } else { + bcx.ins().iconst(ty, 0) + } +} + +pub(crate) fn type_min_max_value( + bcx: &mut FunctionBuilder<'_>, + ty: Type, + signed: bool, +) -> (Value, Value) { + assert!(ty.is_int()); + + if ty == types::I128 { + if signed { + let min = i128::MIN as u128; + let min_lsb = bcx.ins().iconst(types::I64, min as u64 as i64); + let min_msb = bcx.ins().iconst(types::I64, (min >> 64) as u64 as i64); + let min = bcx.ins().iconcat(min_lsb, min_msb); + + let max = i128::MAX as u128; + let max_lsb = bcx.ins().iconst(types::I64, max as u64 as i64); + let max_msb = bcx.ins().iconst(types::I64, (max >> 64) as u64 as i64); + let max = bcx.ins().iconcat(max_lsb, max_msb); + + return (min, max); + } else { + let min_half = bcx.ins().iconst(types::I64, 0); + let min = bcx.ins().iconcat(min_half, min_half); + + let max_half = bcx.ins().iconst(types::I64, u64::MAX as i64); + let max = bcx.ins().iconcat(max_half, max_half); + + return (min, max); + } + } + + let min = match (ty, signed) { + (types::I8, false) | (types::I16, false) | (types::I32, false) | (types::I64, false) => { + 0i64 + } + (types::I8, true) => i64::from(i8::MIN as u8), + (types::I16, true) => i64::from(i16::MIN as u16), + (types::I32, true) => i64::from(i32::MIN as u32), + (types::I64, true) => i64::MIN, + _ => unreachable!(), + }; + + let max = match (ty, signed) { + (types::I8, false) => i64::from(u8::MAX), + (types::I16, false) => i64::from(u16::MAX), + (types::I32, false) => i64::from(u32::MAX), + (types::I64, false) => u64::MAX as i64, + (types::I8, true) => i64::from(i8::MAX as u8), + (types::I16, true) => i64::from(i16::MAX as u16), + (types::I32, true) => i64::from(i32::MAX as u32), + (types::I64, true) => i64::MAX, + _ => unreachable!(), + }; + + let (min, max) = (bcx.ins().iconst(ty, min), bcx.ins().iconst(ty, max)); + + (min, max) +} + +pub(crate) fn type_sign(ty: Ty<'_>) -> bool { + match ty.kind() { + ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..) | ty::Char | ty::Uint(..) | ty::Bool => false, + ty::Int(..) => true, + ty::Float(..) => false, // `signed` is unused for floats + _ => panic!("{}", ty), + } +} + +pub(crate) fn create_wrapper_function( + module: &mut dyn Module, + sig: Signature, + wrapper_name: &str, + callee_name: &str, +) { + let wrapper_func_id = module.declare_function(wrapper_name, Linkage::Export, &sig).unwrap(); + let callee_func_id = module.declare_function(callee_name, Linkage::Import, &sig).unwrap(); + + let mut ctx = Context::new(); + ctx.func.signature = sig; + { + let mut func_ctx = FunctionBuilderContext::new(); + let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let block = bcx.create_block(); + bcx.switch_to_block(block); + let func = &mut bcx.func.stencil; + let args = func + .signature + .params + .iter() + .map(|param| func.dfg.append_block_param(block, param.value_type)) + .collect::<Vec<Value>>(); + + let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func); + let call_inst = bcx.ins().call(callee_func_ref, &args); + let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error + + bcx.ins().return_(&results); + bcx.seal_all_blocks(); + bcx.finalize(); + } + module.define_function(wrapper_func_id, &mut ctx).unwrap(); +} + +pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { + pub(crate) cx: &'clif mut crate::CodegenCx, + pub(crate) module: &'m mut dyn Module, + pub(crate) tcx: TyCtxt<'tcx>, + pub(crate) target_config: TargetFrontendConfig, // Cached from module + pub(crate) pointer_type: Type, // Cached from module + pub(crate) constants_cx: ConstantCx, + pub(crate) func_debug_cx: Option<FunctionDebugContext>, + + pub(crate) instance: Instance<'tcx>, + pub(crate) symbol_name: String, + pub(crate) mir: &'tcx Body<'tcx>, + pub(crate) fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, + + pub(crate) bcx: FunctionBuilder<'clif>, + pub(crate) block_map: IndexVec<BasicBlock, Block>, + pub(crate) local_map: IndexVec<Local, CPlace<'tcx>>, + + /// When `#[track_caller]` is used, the implicit caller location is stored in this variable. + pub(crate) caller_location: Option<CValue<'tcx>>, + + pub(crate) clif_comments: crate::pretty_clif::CommentWriter, + + /// This should only be accessed by `CPlace::new_var`. + pub(crate) next_ssa_var: u32, +} + +impl<'tcx> LayoutOfHelpers<'tcx> for FunctionCx<'_, '_, 'tcx> { + type LayoutOfResult = TyAndLayout<'tcx>; + + #[inline] + fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { + RevealAllLayoutCx(self.tcx).handle_layout_err(err, span, ty) + } +} + +impl<'tcx> FnAbiOfHelpers<'tcx> for FunctionCx<'_, '_, 'tcx> { + type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; + + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + RevealAllLayoutCx(self.tcx).handle_fn_abi_err(err, span, fn_abi_request) + } +} + +impl<'tcx> layout::HasTyCtxt<'tcx> for FunctionCx<'_, '_, 'tcx> { + fn tcx<'b>(&'b self) -> TyCtxt<'tcx> { + self.tcx + } +} + +impl<'tcx> rustc_target::abi::HasDataLayout for FunctionCx<'_, '_, 'tcx> { + fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout { + &self.tcx.data_layout + } +} + +impl<'tcx> layout::HasParamEnv<'tcx> for FunctionCx<'_, '_, 'tcx> { + fn param_env(&self) -> ParamEnv<'tcx> { + ParamEnv::reveal_all() + } +} + +impl<'tcx> HasTargetSpec for FunctionCx<'_, '_, 'tcx> { + fn target_spec(&self) -> &Target { + &self.tcx.sess.target + } +} + +impl<'tcx> FunctionCx<'_, '_, 'tcx> { + pub(crate) fn monomorphize<T>(&self, value: T) -> T + where + T: TypeFoldable<TyCtxt<'tcx>> + Copy, + { + self.instance.instantiate_mir_and_normalize_erasing_regions( + self.tcx, + ty::ParamEnv::reveal_all(), + ty::EarlyBinder::bind(value), + ) + } + + pub(crate) fn clif_type(&self, ty: Ty<'tcx>) -> Option<Type> { + clif_type_from_ty(self.tcx, ty) + } + + pub(crate) fn clif_pair_type(&self, ty: Ty<'tcx>) -> Option<(Type, Type)> { + clif_pair_type_from_ty(self.tcx, ty) + } + + pub(crate) fn get_block(&self, bb: BasicBlock) -> Block { + *self.block_map.get(bb).unwrap() + } + + pub(crate) fn get_local_place(&mut self, local: Local) -> CPlace<'tcx> { + *self.local_map.get(local).unwrap_or_else(|| { + panic!("Local {:?} doesn't exist", local); + }) + } + + pub(crate) fn create_stack_slot(&mut self, size: u32, align: u32) -> Pointer { + let abi_align = if self.tcx.sess.target.arch == "s390x" { 8 } else { 16 }; + if align <= abi_align { + let stack_slot = self.bcx.create_sized_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + // FIXME Don't force the size to a multiple of <abi_align> bytes once Cranelift gets + // a way to specify stack slot alignment. + size: (size + abi_align - 1) / abi_align * abi_align, + align_shift: 4, + }); + Pointer::stack_slot(stack_slot) + } else { + // Alignment is too big to handle using the above hack. Dynamically realign a stack slot + // instead. This wastes some space for the realignment. + let stack_slot = self.bcx.create_sized_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + // FIXME Don't force the size to a multiple of <abi_align> bytes once Cranelift gets + // a way to specify stack slot alignment. + size: (size + align) / abi_align * abi_align, + align_shift: 4, + }); + let base_ptr = self.bcx.ins().stack_addr(self.pointer_type, stack_slot, 0); + let misalign_offset = self.bcx.ins().urem_imm(base_ptr, i64::from(align)); + let realign_offset = self.bcx.ins().irsub_imm(misalign_offset, i64::from(align)); + Pointer::new(self.bcx.ins().iadd(base_ptr, realign_offset)) + } + } + + pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) { + if let Some(debug_context) = &mut self.cx.debug_context { + let (file_id, line, column) = + debug_context.get_span_loc(self.tcx, self.mir.span, source_info.span); + + let source_loc = + self.func_debug_cx.as_mut().unwrap().add_dbg_loc(file_id, line, column); + self.bcx.set_srcloc(source_loc); + } + } + + pub(crate) fn get_caller_location(&mut self, source_info: mir::SourceInfo) -> CValue<'tcx> { + self.mir.caller_location_span(source_info, self.caller_location, self.tcx, |span| { + let const_loc = self.tcx.span_as_caller_location(span); + crate::constant::codegen_const_value(self, const_loc, self.tcx.caller_location_ty()) + }) + } + + pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value { + let mut data = DataDescription::new(); + data.define(msg.as_bytes().to_vec().into_boxed_slice()); + let msg_id = self.module.declare_anonymous_data(false, false).unwrap(); + + // Ignore DuplicateDefinition error, as the data will be the same + let _ = self.module.define_data(msg_id, &data); + + let local_msg_id = self.module.declare_data_in_func(msg_id, self.bcx.func); + if self.clif_comments.enabled() { + self.add_comment(local_msg_id, msg); + } + self.bcx.ins().global_value(self.pointer_type, local_msg_id) + } +} + +pub(crate) struct RevealAllLayoutCx<'tcx>(pub(crate) TyCtxt<'tcx>); + +impl<'tcx> LayoutOfHelpers<'tcx> for RevealAllLayoutCx<'tcx> { + type LayoutOfResult = TyAndLayout<'tcx>; + + #[inline] + fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { + if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + self.0.sess.dcx().span_fatal(span, err.to_string()) + } else { + self.0 + .sess + .dcx() + .span_fatal(span, format!("failed to get layout for `{}`: {}", ty, err)) + } + } +} + +impl<'tcx> FnAbiOfHelpers<'tcx> for RevealAllLayoutCx<'tcx> { + type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; + + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { + self.0.sess.dcx().emit_fatal(Spanned { span, node: err }) + } else { + match fn_abi_request { + FnAbiRequest::OfFnPtr { sig, extra_args } => { + span_bug!(span, "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}"); + } + FnAbiRequest::OfInstance { instance, extra_args } => { + span_bug!( + span, + "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}" + ); + } + } + } + } +} + +impl<'tcx> layout::HasTyCtxt<'tcx> for RevealAllLayoutCx<'tcx> { + fn tcx<'b>(&'b self) -> TyCtxt<'tcx> { + self.0 + } +} + +impl<'tcx> rustc_target::abi::HasDataLayout for RevealAllLayoutCx<'tcx> { + fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout { + &self.0.data_layout + } +} + +impl<'tcx> layout::HasParamEnv<'tcx> for RevealAllLayoutCx<'tcx> { + fn param_env(&self) -> ParamEnv<'tcx> { + ParamEnv::reveal_all() + } +} + +impl<'tcx> HasTargetSpec for RevealAllLayoutCx<'tcx> { + fn target_spec(&self) -> &Target { + &self.0.sess.target + } +} diff --git a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs new file mode 100644 index 00000000000..4154a62234c --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs @@ -0,0 +1,70 @@ +#[cfg(all(unix, feature = "jit"))] +use std::ffi::c_int; +#[cfg(feature = "jit")] +use std::ffi::c_void; + +// FIXME replace with core::ffi::c_size_t once stablized +#[allow(non_camel_case_types)] +#[cfg(feature = "jit")] +type size_t = usize; + +macro_rules! builtin_functions { + ( + $register:ident; + $( + $(#[$attr:meta])? + fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty; + )* + ) => { + #[cfg(feature = "jit")] + #[allow(improper_ctypes)] + extern "C" { + $( + $(#[$attr])? + fn $name($($arg_name: $arg_ty),*) -> $ret_ty; + )* + } + + #[cfg(feature = "jit")] + pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) { + for (name, val) in [$($(#[$attr])? (stringify!($name), $name as *const u8)),*] { + builder.symbol(name, val); + } + } + }; +} + +builtin_functions! { + register_functions_for_jit; + + // integers + fn __muloti4(n: i128, d: i128, oflow: &mut i32) -> i128; + fn __udivti3(n: u128, d: u128) -> u128; + fn __divti3(n: i128, d: i128) -> i128; + fn __umodti3(n: u128, d: u128) -> u128; + fn __modti3(n: i128, d: i128) -> i128; + fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool); + + // floats + fn __floattisf(i: i128) -> f32; + fn __floattidf(i: i128) -> f64; + fn __floatuntisf(i: u128) -> f32; + fn __floatuntidf(i: u128) -> f64; + fn __fixsfti(f: f32) -> i128; + fn __fixdfti(f: f64) -> i128; + fn __fixunssfti(f: f32) -> u128; + fn __fixunsdfti(f: f64) -> u128; + + // allocator + // NOTE: These need to be mentioned here despite not being part of compiler_builtins because + // newer glibc resolve dlsym("malloc") to libc.so despite the override in the rustc binary to + // use jemalloc. Libraries opened with dlopen still get the jemalloc version, causing multiple + // allocators to be mixed, resulting in a crash. + fn calloc(nobj: size_t, size: size_t) -> *mut c_void; + #[cfg(unix)] + fn posix_memalign(memptr: *mut *mut c_void, align: size_t, size: size_t) -> c_int; + fn malloc(size: size_t) -> *mut c_void; + fn realloc(p: *mut c_void, size: size_t) -> *mut c_void; + fn free(p: *mut c_void) -> (); + +} diff --git a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs new file mode 100644 index 00000000000..2093b49ff31 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs @@ -0,0 +1,207 @@ +use std::sync::{Arc, Condvar, Mutex}; + +use jobserver::HelperThread; +use rustc_errors::DiagCtxtHandle; +use rustc_session::Session; + +// FIXME don't panic when a worker thread panics + +pub(super) struct ConcurrencyLimiter { + helper_thread: Option<Mutex<HelperThread>>, + state: Arc<Mutex<state::ConcurrencyLimiterState>>, + available_token_condvar: Arc<Condvar>, + finished: bool, +} + +impl ConcurrencyLimiter { + pub(super) fn new(sess: &Session, pending_jobs: usize) -> Self { + let state = Arc::new(Mutex::new(state::ConcurrencyLimiterState::new(pending_jobs))); + let available_token_condvar = Arc::new(Condvar::new()); + + let state_helper = state.clone(); + let available_token_condvar_helper = available_token_condvar.clone(); + let helper_thread = sess + .jobserver + .clone() + .into_helper_thread(move |token| { + let mut state = state_helper.lock().unwrap(); + match token { + Ok(token) => { + state.add_new_token(token); + available_token_condvar_helper.notify_one(); + } + Err(err) => { + state.poison(format!("failed to acquire jobserver token: {}", err)); + // Notify all threads waiting for a token to give them a chance to + // gracefully exit. + available_token_condvar_helper.notify_all(); + } + } + }) + .unwrap(); + ConcurrencyLimiter { + helper_thread: Some(Mutex::new(helper_thread)), + state, + available_token_condvar, + finished: false, + } + } + + pub(super) fn acquire(&self, dcx: DiagCtxtHandle<'_>) -> ConcurrencyLimiterToken { + let mut state = self.state.lock().unwrap(); + loop { + state.assert_invariants(); + + match state.try_start_job() { + Ok(true) => { + return ConcurrencyLimiterToken { + state: self.state.clone(), + available_token_condvar: self.available_token_condvar.clone(), + }; + } + Ok(false) => {} + Err(err) => { + // An error happened when acquiring the token. Raise it as fatal error. + // Make sure to drop the mutex guard first to prevent poisoning the mutex. + drop(state); + if let Some(err) = err { + dcx.fatal(err); + } else { + // The error was already emitted, but compilation continued. Raise a silent + // fatal error. + rustc_errors::FatalError.raise(); + } + } + } + + self.helper_thread.as_ref().unwrap().lock().unwrap().request_token(); + state = self.available_token_condvar.wait(state).unwrap(); + } + } + + pub(crate) fn finished(mut self) { + self.helper_thread.take(); + + // Assert that all jobs have finished + let state = Mutex::get_mut(Arc::get_mut(&mut self.state).unwrap()).unwrap(); + state.assert_done(); + + self.finished = true; + } +} + +impl Drop for ConcurrencyLimiter { + fn drop(&mut self) { + if !self.finished && !std::thread::panicking() { + panic!("Forgot to call finished() on ConcurrencyLimiter"); + } + } +} + +#[derive(Debug)] +pub(super) struct ConcurrencyLimiterToken { + state: Arc<Mutex<state::ConcurrencyLimiterState>>, + available_token_condvar: Arc<Condvar>, +} + +impl Drop for ConcurrencyLimiterToken { + fn drop(&mut self) { + let mut state = self.state.lock().unwrap(); + state.job_finished(); + self.available_token_condvar.notify_one(); + } +} + +mod state { + use jobserver::Acquired; + + #[derive(Debug)] + pub(super) struct ConcurrencyLimiterState { + pending_jobs: usize, + active_jobs: usize, + + poisoned: bool, + stored_error: Option<String>, + + // None is used to represent the implicit token, Some to represent explicit tokens + tokens: Vec<Option<Acquired>>, + } + + impl ConcurrencyLimiterState { + pub(super) fn new(pending_jobs: usize) -> Self { + ConcurrencyLimiterState { + pending_jobs, + active_jobs: 0, + poisoned: false, + stored_error: None, + tokens: vec![None], + } + } + + pub(super) fn assert_invariants(&self) { + // There must be no excess active jobs + assert!(self.active_jobs <= self.pending_jobs); + + // There may not be more active jobs than there are tokens + assert!(self.active_jobs <= self.tokens.len()); + } + + pub(super) fn assert_done(&self) { + assert_eq!(self.pending_jobs, 0); + assert_eq!(self.active_jobs, 0); + } + + pub(super) fn add_new_token(&mut self, token: Acquired) { + self.tokens.push(Some(token)); + self.drop_excess_capacity(); + } + + pub(super) fn try_start_job(&mut self) -> Result<bool, Option<String>> { + if self.poisoned { + return Err(self.stored_error.take()); + } + + if self.active_jobs < self.tokens.len() { + // Using existing token + self.job_started(); + return Ok(true); + } + + Ok(false) + } + + pub(super) fn job_started(&mut self) { + self.assert_invariants(); + self.active_jobs += 1; + self.drop_excess_capacity(); + self.assert_invariants(); + } + + pub(super) fn job_finished(&mut self) { + self.assert_invariants(); + self.pending_jobs -= 1; + self.active_jobs -= 1; + self.assert_invariants(); + self.drop_excess_capacity(); + self.assert_invariants(); + } + + pub(super) fn poison(&mut self, error: String) { + self.poisoned = true; + self.stored_error = Some(error); + } + + fn drop_excess_capacity(&mut self) { + self.assert_invariants(); + + // Drop all tokens that can never be used anymore + self.tokens.truncate(std::cmp::max(self.pending_jobs, 1)); + + // Keep some excess tokens to satisfy requests faster + const MAX_EXTRA_CAPACITY: usize = 2; + self.tokens.truncate(std::cmp::max(self.active_jobs + MAX_EXTRA_CAPACITY, 1)); + + self.assert_invariants(); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/config.rs b/compiler/rustc_codegen_cranelift/src/config.rs new file mode 100644 index 00000000000..12bce680d9e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/config.rs @@ -0,0 +1,109 @@ +use std::env; +use std::str::FromStr; + +fn bool_env_var(key: &str) -> bool { + env::var(key).as_deref() == Ok("1") +} + +/// The mode to use for compilation. +#[derive(Copy, Clone, Debug)] +pub enum CodegenMode { + /// AOT compile the crate. This is the default. + Aot, + /// JIT compile and execute the crate. + Jit, + /// JIT compile and execute the crate, but only compile functions the first time they are used. + JitLazy, +} + +impl FromStr for CodegenMode { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "aot" => Ok(CodegenMode::Aot), + "jit" => Ok(CodegenMode::Jit), + "jit-lazy" => Ok(CodegenMode::JitLazy), + _ => Err(format!("Unknown codegen mode `{}`", s)), + } + } +} + +/// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars. +#[derive(Clone, Debug)] +pub struct BackendConfig { + /// Should the crate be AOT compiled or JIT executed. + /// + /// Defaults to AOT compilation. Can be set using `-Cllvm-args=mode=...`. + pub codegen_mode: CodegenMode, + + /// When JIT mode is enable pass these arguments to the program. + /// + /// Defaults to the value of `CG_CLIF_JIT_ARGS`. + pub jit_args: Vec<String>, + + /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run + /// once before passing the clif ir to Cranelift for compilation. + /// + /// Defaults to true when the `CG_CLIF_ENABLE_VERIFIER` env var is set to 1 or when cg_clif is + /// compiled with debug assertions enabled or false otherwise. Can be set using + /// `-Cllvm-args=enable_verifier=...`. + pub enable_verifier: bool, + + /// Don't cache object files in the incremental cache. Useful during development of cg_clif + /// to make it possible to use incremental mode for all analyses performed by rustc without + /// caching object files when their content should have been changed by a change to cg_clif. + /// + /// Defaults to true when the `CG_CLIF_DISABLE_INCR_CACHE` env var is set to 1 or false + /// otherwise. Can be set using `-Cllvm-args=disable_incr_cache=...`. + pub disable_incr_cache: bool, +} + +impl Default for BackendConfig { + fn default() -> Self { + BackendConfig { + codegen_mode: CodegenMode::Aot, + jit_args: { + match std::env::var("CG_CLIF_JIT_ARGS") { + Ok(args) => args.split(' ').map(|arg| arg.to_string()).collect(), + Err(std::env::VarError::NotPresent) => vec![], + Err(std::env::VarError::NotUnicode(s)) => { + panic!("CG_CLIF_JIT_ARGS not unicode: {:?}", s); + } + } + }, + enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"), + disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"), + } + } +} + +impl BackendConfig { + /// Parse the configuration passed in using `-Cllvm-args`. + pub fn from_opts(opts: &[String]) -> Result<Self, String> { + fn parse_bool(name: &str, value: &str) -> Result<bool, String> { + value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name)) + } + + let mut config = BackendConfig::default(); + for opt in opts { + if opt.starts_with("-import-instr-limit") { + // Silently ignore -import-instr-limit. It is set by rust's build system even when + // testing cg_clif. + continue; + } + if let Some((name, value)) = opt.split_once('=') { + match name { + "mode" => config.codegen_mode = value.parse()?, + "enable_verifier" => config.enable_verifier = parse_bool(name, value)?, + "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?, + _ => return Err(format!("Unknown option `{}`", name)), + } + } else { + return Err(format!("Invalid option `{}`", opt)); + } + } + + Ok(config) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs new file mode 100644 index 00000000000..0ba163f50ae --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -0,0 +1,605 @@ +//! Handling of `static`s, `const`s and promoted allocations + +use std::cmp::Ordering; + +use cranelift_module::*; +use rustc_data_structures::fx::FxHashSet; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir::interpret::{read_target_uint, AllocId, GlobalAlloc, Scalar}; +use rustc_middle::ty::{Binder, ExistentialTraitRef, ScalarInt}; + +use crate::prelude::*; + +pub(crate) struct ConstantCx { + todo: Vec<TodoItem>, + anon_allocs: FxHashMap<AllocId, DataId>, +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +enum TodoItem { + Alloc(AllocId), + Static(DefId), +} + +impl ConstantCx { + pub(crate) fn new() -> Self { + ConstantCx { todo: vec![], anon_allocs: FxHashMap::default() } + } + + pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) { + define_all_allocs(tcx, module, &mut self); + } +} + +pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) -> DataId { + let mut constants_cx = ConstantCx::new(); + constants_cx.todo.push(TodoItem::Static(def_id)); + constants_cx.finalize(tcx, module); + + data_id_for_static( + tcx, module, def_id, false, + // For a declaration the stated mutability doesn't matter. + false, + ) +} + +pub(crate) fn codegen_tls_ref<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + def_id: DefId, + layout: TyAndLayout<'tcx>, +) -> CValue<'tcx> { + let tls_ptr = if !def_id.is_local() && fx.tcx.needs_thread_local_shim(def_id) { + let instance = ty::Instance { + def: ty::InstanceKind::ThreadLocalShim(def_id), + args: ty::GenericArgs::empty(), + }; + let func_ref = fx.get_function_ref(instance); + let call = fx.bcx.ins().call(func_ref, &[]); + fx.bcx.func.dfg.first_result(call) + } else { + let data_id = data_id_for_static( + fx.tcx, fx.module, def_id, false, + // For a declaration the stated mutability doesn't matter. + false, + ); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("tls {:?}", def_id)); + } + fx.bcx.ins().tls_value(fx.pointer_type, local_data_id) + }; + CValue::by_val(tls_ptr, layout) +} + +pub(crate) fn eval_mir_constant<'tcx>( + fx: &FunctionCx<'_, '_, 'tcx>, + constant: &ConstOperand<'tcx>, +) -> (ConstValue<'tcx>, Ty<'tcx>) { + let cv = fx.monomorphize(constant.const_); + // This cannot fail because we checked all required_consts in advance. + let val = cv + .eval(fx.tcx, ty::ParamEnv::reveal_all(), constant.span) + .expect("erroneous constant missed by mono item collection"); + (val, cv.ty()) +} + +pub(crate) fn codegen_constant_operand<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + constant: &ConstOperand<'tcx>, +) -> CValue<'tcx> { + let (const_val, ty) = eval_mir_constant(fx, constant); + codegen_const_value(fx, const_val, ty) +} + +pub(crate) fn codegen_const_value<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + const_val: ConstValue<'tcx>, + ty: Ty<'tcx>, +) -> CValue<'tcx> { + let layout = fx.layout_of(ty); + assert!(layout.is_sized(), "unsized const value"); + + if layout.is_zst() { + return CValue::zst(layout); + } + + match const_val { + ConstValue::ZeroSized => unreachable!(), // we already handled ZST above + ConstValue::Scalar(x) => match x { + Scalar::Int(int) => { + if fx.clif_type(layout.ty).is_some() { + return CValue::const_val(fx, layout, int); + } else { + let raw_val = int.size().truncate(int.to_bits(int.size())); + let val = match int.size().bytes() { + 1 => fx.bcx.ins().iconst(types::I8, raw_val as i64), + 2 => fx.bcx.ins().iconst(types::I16, raw_val as i64), + 4 => fx.bcx.ins().iconst(types::I32, raw_val as i64), + 8 => fx.bcx.ins().iconst(types::I64, raw_val as i64), + 16 => { + let lsb = fx.bcx.ins().iconst(types::I64, raw_val as u64 as i64); + let msb = + fx.bcx.ins().iconst(types::I64, (raw_val >> 64) as u64 as i64); + fx.bcx.ins().iconcat(lsb, msb) + } + _ => unreachable!(), + }; + + // FIXME avoid this extra copy to the stack and directly write to the final + // destination + let place = CPlace::new_stack_slot(fx, layout); + place.to_ptr().store(fx, val, MemFlags::trusted()); + place.to_cvalue(fx) + } + } + Scalar::Ptr(ptr, _size) => { + let (prov, offset) = ptr.into_parts(); // we know the `offset` is relative + let alloc_id = prov.alloc_id(); + let base_addr = match fx.tcx.global_alloc(alloc_id) { + GlobalAlloc::Memory(alloc) => { + if alloc.inner().len() == 0 { + assert_eq!(offset, Size::ZERO); + fx.bcx.ins().iconst(fx.pointer_type, alloc.inner().align.bytes() as i64) + } else { + let data_id = data_id_for_alloc_id( + &mut fx.constants_cx, + fx.module, + alloc_id, + alloc.inner().mutability, + ); + let local_data_id = + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", alloc_id)); + } + fx.bcx.ins().global_value(fx.pointer_type, local_data_id) + } + } + GlobalAlloc::Function { instance, .. } => { + let func_id = crate::abi::import_function(fx.tcx, fx.module, instance); + let local_func_id = + fx.module.declare_func_in_func(func_id, &mut fx.bcx.func); + fx.bcx.ins().func_addr(fx.pointer_type, local_func_id) + } + GlobalAlloc::VTable(ty, trait_ref) => { + let data_id = data_id_for_vtable( + fx.tcx, + &mut fx.constants_cx, + fx.module, + ty, + trait_ref, + ); + let local_data_id = + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.bcx.ins().global_value(fx.pointer_type, local_data_id) + } + GlobalAlloc::Static(def_id) => { + assert!(fx.tcx.is_static(def_id)); + let data_id = data_id_for_static( + fx.tcx, fx.module, def_id, false, + // For a declaration the stated mutability doesn't matter. + false, + ); + let local_data_id = + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", def_id)); + } + fx.bcx.ins().global_value(fx.pointer_type, local_data_id) + } + }; + let val = if offset.bytes() != 0 { + fx.bcx.ins().iadd_imm(base_addr, i64::try_from(offset.bytes()).unwrap()) + } else { + base_addr + }; + CValue::by_val(val, layout) + } + }, + ConstValue::Indirect { alloc_id, offset } => CValue::by_ref( + pointer_for_allocation(fx, alloc_id) + .offset_i64(fx, i64::try_from(offset.bytes()).unwrap()), + layout, + ), + ConstValue::Slice { data, meta } => { + let alloc_id = fx.tcx.reserve_and_set_memory_alloc(data); + let ptr = pointer_for_allocation(fx, alloc_id).get_addr(fx); + let len = fx.bcx.ins().iconst(fx.pointer_type, meta as i64); + CValue::by_val_pair(ptr, len, layout) + } + } +} + +fn pointer_for_allocation<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + alloc_id: AllocId, +) -> crate::pointer::Pointer { + let alloc = fx.tcx.global_alloc(alloc_id).unwrap_memory(); + let data_id = + data_id_for_alloc_id(&mut fx.constants_cx, fx.module, alloc_id, alloc.inner().mutability); + + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", alloc_id)); + } + let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); + crate::pointer::Pointer::new(global_ptr) +} + +pub(crate) fn data_id_for_alloc_id( + cx: &mut ConstantCx, + module: &mut dyn Module, + alloc_id: AllocId, + mutability: rustc_hir::Mutability, +) -> DataId { + cx.todo.push(TodoItem::Alloc(alloc_id)); + *cx.anon_allocs + .entry(alloc_id) + .or_insert_with(|| module.declare_anonymous_data(mutability.is_mut(), false).unwrap()) +} + +pub(crate) fn data_id_for_vtable<'tcx>( + tcx: TyCtxt<'tcx>, + cx: &mut ConstantCx, + module: &mut dyn Module, + ty: Ty<'tcx>, + trait_ref: Option<Binder<'tcx, ExistentialTraitRef<'tcx>>>, +) -> DataId { + let alloc_id = tcx.vtable_allocation((ty, trait_ref)); + data_id_for_alloc_id(cx, module, alloc_id, Mutability::Not) +} + +fn data_id_for_static( + tcx: TyCtxt<'_>, + module: &mut dyn Module, + def_id: DefId, + definition: bool, + definition_writable: bool, +) -> DataId { + let attrs = tcx.codegen_fn_attrs(def_id); + + let instance = Instance::mono(tcx, def_id); + let symbol_name = tcx.symbol_name(instance).name; + + if let Some(import_linkage) = attrs.import_linkage { + assert!(!definition); + assert!(!tcx.is_mutable_static(def_id)); + + let ty = instance.ty(tcx, ParamEnv::reveal_all()); + let align = tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().align.pref.bytes(); + + let linkage = if import_linkage == rustc_middle::mir::mono::Linkage::ExternalWeak + || import_linkage == rustc_middle::mir::mono::Linkage::WeakAny + { + Linkage::Preemptible + } else { + Linkage::Import + }; + + let data_id = match module.declare_data( + symbol_name, + linkage, + false, + attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL), + ) { + Ok(data_id) => data_id, + Err(ModuleError::IncompatibleDeclaration(_)) => tcx.dcx().fatal(format!( + "attempt to declare `{symbol_name}` as static, but it was already declared as function" + )), + Err(err) => Err::<_, _>(err).unwrap(), + }; + + // Comment copied from https://github.com/rust-lang/rust/blob/45060c2a66dfd667f88bd8b94261b28a58d85bd5/src/librustc_codegen_llvm/consts.rs#L141 + // Declare an internal global `extern_with_linkage_foo` which + // is initialized with the address of `foo`. If `foo` is + // discarded during linking (for example, if `foo` has weak + // linkage and there are no definitions), then + // `extern_with_linkage_foo` will instead be initialized to + // zero. + + let ref_name = format!("_rust_extern_with_linkage_{}", symbol_name); + let ref_data_id = module.declare_data(&ref_name, Linkage::Local, false, false).unwrap(); + let mut data = DataDescription::new(); + data.set_align(align); + let data_gv = module.declare_data_in_data(data_id, &mut data); + data.define(std::iter::repeat(0).take(pointer_ty(tcx).bytes() as usize).collect()); + data.write_data_addr(0, data_gv, 0); + match module.define_data(ref_data_id, &data) { + // Every time the static is referenced there will be another definition of this global, + // so duplicate definitions are expected and allowed. + Err(ModuleError::DuplicateDefinition(_)) => {} + res => res.unwrap(), + } + + return ref_data_id; + } + + let linkage = if definition { + crate::linkage::get_static_linkage(tcx, def_id) + } else if attrs.linkage == Some(rustc_middle::mir::mono::Linkage::ExternalWeak) + || attrs.linkage == Some(rustc_middle::mir::mono::Linkage::WeakAny) + { + Linkage::Preemptible + } else { + Linkage::Import + }; + + let data_id = match module.declare_data( + symbol_name, + linkage, + definition_writable, + attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL), + ) { + Ok(data_id) => data_id, + Err(ModuleError::IncompatibleDeclaration(_)) => tcx.dcx().fatal(format!( + "attempt to declare `{symbol_name}` as static, but it was already declared as function" + )), + Err(err) => Err::<_, _>(err).unwrap(), + }; + + data_id +} + +fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut ConstantCx) { + let mut done = FxHashSet::default(); + while let Some(todo_item) = cx.todo.pop() { + if !done.insert(todo_item) { + continue; + } + + let (data_id, alloc, section_name) = match todo_item { + TodoItem::Alloc(alloc_id) => { + let alloc = match tcx.global_alloc(alloc_id) { + GlobalAlloc::Memory(alloc) => alloc, + GlobalAlloc::Function { .. } + | GlobalAlloc::Static(_) + | GlobalAlloc::VTable(..) => { + unreachable!() + } + }; + // FIXME: should we have a cache so we don't do this multiple times for the same `ConstAllocation`? + let data_id = *cx.anon_allocs.entry(alloc_id).or_insert_with(|| { + module.declare_anonymous_data(alloc.inner().mutability.is_mut(), false).unwrap() + }); + (data_id, alloc, None) + } + TodoItem::Static(def_id) => { + let section_name = tcx.codegen_fn_attrs(def_id).link_section; + + let alloc = tcx.eval_static_initializer(def_id).unwrap(); + + let data_id = data_id_for_static( + tcx, + module, + def_id, + true, + alloc.inner().mutability == Mutability::Mut, + ); + (data_id, alloc, section_name) + } + }; + + let mut data = DataDescription::new(); + let alloc = alloc.inner(); + data.set_align(alloc.align.bytes()); + + if let Some(section_name) = section_name { + let (segment_name, section_name) = if tcx.sess.target.is_like_osx { + // See https://github.com/llvm/llvm-project/blob/main/llvm/lib/MC/MCSectionMachO.cpp + let mut parts = section_name.as_str().split(','); + let Some(segment_name) = parts.next() else { + tcx.dcx().fatal(format!( + "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma", + section_name + )); + }; + let Some(section_name) = parts.next() else { + tcx.dcx().fatal(format!( + "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma", + section_name + )); + }; + if section_name.len() > 16 { + tcx.dcx().fatal(format!( + "#[link_section = \"{}\"] is not valid for macos target: section name bigger than 16 bytes", + section_name + )); + } + let section_type = parts.next().unwrap_or("regular"); + if section_type != "regular" && section_type != "cstring_literals" { + tcx.dcx().fatal(format!( + "#[link_section = \"{}\"] is not supported: unsupported section type {}", + section_name, section_type, + )); + } + let _attrs = parts.next(); + if parts.next().is_some() { + tcx.dcx().fatal(format!( + "#[link_section = \"{}\"] is not valid for macos target: too many components", + section_name + )); + } + // FIXME(bytecodealliance/wasmtime#8901) set S_CSTRING_LITERALS section type when + // cstring_literals is specified + (segment_name, section_name) + } else { + ("", section_name.as_str()) + }; + data.set_segment_section(segment_name, section_name); + } + + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec(); + data.define(bytes.into_boxed_slice()); + + for &(offset, prov) in alloc.provenance().ptrs().iter() { + let alloc_id = prov.alloc_id(); + let addend = { + let endianness = tcx.data_layout.endian; + let offset = offset.bytes() as usize; + let ptr_size = tcx.data_layout.pointer_size; + let bytes = &alloc.inspect_with_uninit_and_ptr_outside_interpreter( + offset..offset + ptr_size.bytes() as usize, + ); + read_target_uint(endianness, bytes).unwrap() + }; + + let reloc_target_alloc = tcx.global_alloc(alloc_id); + let data_id = match reloc_target_alloc { + GlobalAlloc::Function { instance, .. } => { + assert_eq!(addend, 0); + let func_id = + crate::abi::import_function(tcx, module, instance.polymorphize(tcx)); + let local_func_id = module.declare_func_in_data(func_id, &mut data); + data.write_function_addr(offset.bytes() as u32, local_func_id); + continue; + } + GlobalAlloc::Memory(target_alloc) => { + data_id_for_alloc_id(cx, module, alloc_id, target_alloc.inner().mutability) + } + GlobalAlloc::VTable(ty, trait_ref) => { + data_id_for_vtable(tcx, cx, module, ty, trait_ref) + } + GlobalAlloc::Static(def_id) => { + if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) + { + tcx.dcx().fatal(format!( + "Allocation {:?} contains reference to TLS value {:?}", + alloc_id, def_id + )); + } + + // Don't push a `TodoItem::Static` here, as it will cause statics used by + // multiple crates to be duplicated between them. It isn't necessary anyway, + // as it will get pushed by `codegen_static` when necessary. + data_id_for_static( + tcx, module, def_id, false, + // For a declaration the stated mutability doesn't matter. + false, + ) + } + }; + + let global_value = module.declare_data_in_data(data_id, &mut data); + data.write_data_addr(offset.bytes() as u32, global_value, addend as i64); + } + + module.define_data(data_id, &data).unwrap(); + } + + assert!(cx.todo.is_empty(), "{:?}", cx.todo); +} + +/// Used only for intrinsic implementations that need a compile-time constant +pub(crate) fn mir_operand_get_const_val<'tcx>( + fx: &FunctionCx<'_, '_, 'tcx>, + operand: &Operand<'tcx>, +) -> Option<ScalarInt> { + match operand { + Operand::Constant(const_) => eval_mir_constant(fx, const_).0.try_to_scalar_int(), + // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored + // inside a temporary before being passed to the intrinsic requiring the const argument. + // This code tries to find a single constant defining definition of the referenced local. + Operand::Copy(place) | Operand::Move(place) => { + if !place.projection.is_empty() { + return None; + } + let mut computed_scalar_int = None; + for bb_data in fx.mir.basic_blocks.iter() { + for stmt in &bb_data.statements { + match &stmt.kind { + StatementKind::Assign(local_and_rvalue) if &local_and_rvalue.0 == place => { + match &local_and_rvalue.1 { + Rvalue::Cast( + CastKind::IntToInt + | CastKind::FloatToFloat + | CastKind::FloatToInt + | CastKind::IntToFloat + | CastKind::FnPtrToPtr + | CastKind::PtrToPtr, + operand, + ty, + ) => { + if computed_scalar_int.is_some() { + return None; // local assigned twice + } + if !matches!(ty.kind(), ty::Uint(_) | ty::Int(_)) { + return None; + } + let scalar_int = mir_operand_get_const_val(fx, operand)?; + let scalar_int = + match fx.layout_of(*ty).size.cmp(&scalar_int.size()) { + Ordering::Equal => scalar_int, + Ordering::Less => match ty.kind() { + ty::Uint(_) => ScalarInt::try_from_uint( + scalar_int.to_uint(scalar_int.size()), + fx.layout_of(*ty).size, + ) + .unwrap(), + ty::Int(_) => ScalarInt::try_from_int( + scalar_int.to_int(scalar_int.size()), + fx.layout_of(*ty).size, + ) + .unwrap(), + _ => unreachable!(), + }, + Ordering::Greater => return None, + }; + computed_scalar_int = Some(scalar_int); + } + Rvalue::Use(operand) => { + computed_scalar_int = mir_operand_get_const_val(fx, operand) + } + _ => return None, + } + } + StatementKind::SetDiscriminant { place: stmt_place, variant_index: _ } + if &**stmt_place == place => + { + return None; + } + StatementKind::Intrinsic(ref intrinsic) => match **intrinsic { + NonDivergingIntrinsic::CopyNonOverlapping(..) => return None, + NonDivergingIntrinsic::Assume(..) => {} + }, + // conservative handling + StatementKind::Assign(_) + | StatementKind::FakeRead(_) + | StatementKind::SetDiscriminant { .. } + | StatementKind::Deinit(_) + | StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Retag(_, _) + | StatementKind::AscribeUserType(_, _) + | StatementKind::PlaceMention(..) + | StatementKind::Coverage(_) + | StatementKind::ConstEvalCounter + | StatementKind::Nop => {} + } + } + match &bb_data.terminator().kind { + TerminatorKind::Goto { .. } + | TerminatorKind::SwitchInt { .. } + | TerminatorKind::UnwindResume + | TerminatorKind::UnwindTerminate(_) + | TerminatorKind::Return + | TerminatorKind::Unreachable + | TerminatorKind::Drop { .. } + | TerminatorKind::Assert { .. } => {} + TerminatorKind::Yield { .. } + | TerminatorKind::CoroutineDrop + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } => unreachable!(), + TerminatorKind::InlineAsm { .. } => return None, + TerminatorKind::Call { destination, target: Some(_), .. } + if destination == place => + { + return None; + } + TerminatorKind::TailCall { .. } => return None, + TerminatorKind::Call { .. } => {} + } + } + computed_scalar_int + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs new file mode 100644 index 00000000000..36af7d4450d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs @@ -0,0 +1,203 @@ +//! Write the debuginfo into an object file. + +use cranelift_module::{DataId, FuncId}; +use cranelift_object::ObjectProduct; +use gimli::write::{Address, AttributeValue, EndianVec, Result, Sections, Writer}; +use gimli::{RunTimeEndian, SectionId}; +use rustc_data_structures::fx::FxHashMap; + +use super::object::WriteDebugInfo; +use super::DebugContext; + +pub(super) fn address_for_func(func_id: FuncId) -> Address { + let symbol = func_id.as_u32(); + assert!(symbol & 1 << 31 == 0); + Address::Symbol { symbol: symbol as usize, addend: 0 } +} + +pub(super) fn address_for_data(data_id: DataId) -> Address { + let symbol = data_id.as_u32(); + assert!(symbol & 1 << 31 == 0); + Address::Symbol { symbol: (symbol | 1 << 31) as usize, addend: 0 } +} + +impl DebugContext { + pub(crate) fn emit(&mut self, product: &mut ObjectProduct) { + let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone()); + let root = self.dwarf.unit.root(); + let root = self.dwarf.unit.get_mut(root); + root.set(gimli::DW_AT_ranges, AttributeValue::RangeListRef(unit_range_list_id)); + + let mut sections = Sections::new(WriterRelocate::new(self.endian)); + self.dwarf.write(&mut sections).unwrap(); + + let mut section_map = FxHashMap::default(); + let _: Result<()> = sections.for_each_mut(|id, section| { + if !section.writer.slice().is_empty() { + let section_id = product.add_debug_section(id, section.writer.take()); + section_map.insert(id, section_id); + } + Ok(()) + }); + + let _: Result<()> = sections.for_each(|id, section| { + if let Some(section_id) = section_map.get(&id) { + for reloc in §ion.relocs { + product.add_debug_reloc(§ion_map, section_id, reloc); + } + } + Ok(()) + }); + } +} + +#[derive(Clone)] +pub(crate) struct DebugReloc { + pub(crate) offset: u32, + pub(crate) size: u8, + pub(crate) name: DebugRelocName, + pub(crate) addend: i64, + pub(crate) kind: object::RelocationKind, +} + +#[derive(Clone)] +pub(crate) enum DebugRelocName { + Section(SectionId), + Symbol(usize), +} + +/// A [`Writer`] that collects all necessary relocations. +#[derive(Clone)] +pub(super) struct WriterRelocate { + pub(super) relocs: Vec<DebugReloc>, + pub(super) writer: EndianVec<RunTimeEndian>, +} + +impl WriterRelocate { + pub(super) fn new(endian: RunTimeEndian) -> Self { + WriterRelocate { relocs: Vec::new(), writer: EndianVec::new(endian) } + } + + /// Perform the collected relocations to be usable for JIT usage. + #[cfg(all(feature = "jit", not(windows)))] + pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec<u8> { + for reloc in self.relocs.drain(..) { + match reloc.name { + super::DebugRelocName::Section(_) => unreachable!(), + super::DebugRelocName::Symbol(sym) => { + let addr = jit_module.get_finalized_function( + cranelift_module::FuncId::from_u32(sym.try_into().unwrap()), + ); + let val = (addr as u64 as i64 + reloc.addend) as u64; + self.writer.write_udata_at(reloc.offset as usize, val, reloc.size).unwrap(); + } + } + } + self.writer.into_vec() + } +} + +impl Writer for WriterRelocate { + type Endian = RunTimeEndian; + + fn endian(&self) -> Self::Endian { + self.writer.endian() + } + + fn len(&self) -> usize { + self.writer.len() + } + + fn write(&mut self, bytes: &[u8]) -> Result<()> { + self.writer.write(bytes) + } + + fn write_at(&mut self, offset: usize, bytes: &[u8]) -> Result<()> { + self.writer.write_at(offset, bytes) + } + + fn write_address(&mut self, address: Address, size: u8) -> Result<()> { + match address { + Address::Constant(val) => self.write_udata(val, size), + Address::Symbol { symbol, addend } => { + let offset = self.len() as u64; + self.relocs.push(DebugReloc { + offset: offset as u32, + size, + name: DebugRelocName::Symbol(symbol), + addend, + kind: object::RelocationKind::Absolute, + }); + self.write_udata(0, size) + } + } + } + + fn write_offset(&mut self, val: usize, section: SectionId, size: u8) -> Result<()> { + let offset = self.len() as u32; + self.relocs.push(DebugReloc { + offset, + size, + name: DebugRelocName::Section(section), + addend: val as i64, + kind: object::RelocationKind::Absolute, + }); + self.write_udata(0, size) + } + + fn write_offset_at( + &mut self, + offset: usize, + val: usize, + section: SectionId, + size: u8, + ) -> Result<()> { + self.relocs.push(DebugReloc { + offset: offset as u32, + size, + name: DebugRelocName::Section(section), + addend: val as i64, + kind: object::RelocationKind::Absolute, + }); + self.write_udata_at(offset, 0, size) + } + + fn write_eh_pointer(&mut self, address: Address, eh_pe: gimli::DwEhPe, size: u8) -> Result<()> { + match address { + // Address::Constant arm copied from gimli + Address::Constant(val) => { + // Indirect doesn't matter here. + let val = match eh_pe.application() { + gimli::DW_EH_PE_absptr => val, + gimli::DW_EH_PE_pcrel => { + // FIXME better handling of sign + let offset = self.len() as u64; + offset.wrapping_sub(val) + } + _ => { + return Err(gimli::write::Error::UnsupportedPointerEncoding(eh_pe)); + } + }; + self.write_eh_pointer_data(val, eh_pe.format(), size) + } + Address::Symbol { symbol, addend } => match eh_pe.application() { + gimli::DW_EH_PE_pcrel => { + let size = match eh_pe.format() { + gimli::DW_EH_PE_sdata4 => 4, + gimli::DW_EH_PE_sdata8 => 8, + _ => return Err(gimli::write::Error::UnsupportedPointerEncoding(eh_pe)), + }; + self.relocs.push(DebugReloc { + offset: self.len() as u32, + size, + name: DebugRelocName::Symbol(symbol), + addend, + kind: object::RelocationKind::Relative, + }); + self.write_udata(0, size) + } + _ => Err(gimli::write::Error::UnsupportedPointerEncoding(eh_pe)), + }, + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs new file mode 100644 index 00000000000..78b3d5a58f4 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs @@ -0,0 +1,172 @@ +//! Line info generation (`.debug_line`) + +use std::ffi::OsStr; +use std::path::{Component, Path}; + +use cranelift_codegen::binemit::CodeOffset; +use cranelift_codegen::MachSrcLoc; +use gimli::write::{AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable}; +use rustc_span::{ + hygiene, FileName, Pos, SourceFile, SourceFileAndLine, SourceFileHash, SourceFileHashAlgorithm, +}; + +use crate::debuginfo::emit::address_for_func; +use crate::debuginfo::FunctionDebugContext; +use crate::prelude::*; + +// OPTIMIZATION: It is cheaper to do this in one pass than using `.parent()` and `.file_name()`. +fn split_path_dir_and_file(path: &Path) -> (&Path, &OsStr) { + let mut iter = path.components(); + let file_name = match iter.next_back() { + Some(Component::Normal(p)) => p, + component => { + panic!( + "Path component {:?} of path {} is an invalid filename", + component, + path.display() + ); + } + }; + let parent = iter.as_path(); + (parent, file_name) +} + +// OPTIMIZATION: Avoid UTF-8 validation on UNIX. +fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + path.as_bytes() + } + #[cfg(not(unix))] + { + path.to_str().unwrap().as_bytes() + } +} + +const MD5_LEN: usize = 16; + +fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> { + if hash.kind == SourceFileHashAlgorithm::Md5 { + let mut buf = [0u8; MD5_LEN]; + buf.copy_from_slice(hash.hash_bytes()); + Some(FileInfo { timestamp: 0, size: 0, md5: buf }) + } else { + None + } +} + +impl DebugContext { + pub(crate) fn get_span_loc( + &mut self, + tcx: TyCtxt<'_>, + function_span: Span, + span: Span, + ) -> (FileId, u64, u64) { + // Match behavior of `FunctionCx::adjusted_span_and_dbg_scope`. + let span = hygiene::walk_chain_collapsed(span, function_span); + match tcx.sess.source_map().lookup_line(span.lo()) { + Ok(SourceFileAndLine { sf: file, line }) => { + let file_id = self.add_source_file(&file); + let line_pos = file.lines()[line]; + let col = file.relative_position(span.lo()) - line_pos; + + (file_id, u64::try_from(line).unwrap() + 1, u64::from(col.to_u32()) + 1) + } + Err(file) => (self.add_source_file(&file), 0, 0), + } + } + + pub(crate) fn add_source_file(&mut self, source_file: &SourceFile) -> FileId { + let cache_key = (source_file.stable_id, source_file.src_hash); + *self.created_files.entry(cache_key).or_insert_with(|| { + let line_program: &mut LineProgram = &mut self.dwarf.unit.line_program; + let line_strings: &mut LineStringTable = &mut self.dwarf.line_strings; + + match &source_file.name { + FileName::Real(path) => { + let (dir_path, file_name) = + split_path_dir_and_file(path.to_path(self.filename_display_preference)); + let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str()); + let file_name = osstr_as_utf8_bytes(file_name); + + let dir_id = if !dir_name.is_empty() { + let dir_name = + LineString::new(dir_name, line_program.encoding(), line_strings); + line_program.add_directory(dir_name) + } else { + line_program.default_directory() + }; + let file_name = + LineString::new(file_name, line_program.encoding(), line_strings); + + let info = make_file_info(source_file.src_hash); + + line_program.file_has_md5 &= info.is_some(); + line_program.add_file(file_name, dir_id, info) + } + filename => { + let dir_id = line_program.default_directory(); + let dummy_file_name = LineString::new( + filename.display(self.filename_display_preference).to_string().into_bytes(), + line_program.encoding(), + line_strings, + ); + line_program.add_file(dummy_file_name, dir_id, None) + } + } + }) + } +} + +impl FunctionDebugContext { + pub(crate) fn add_dbg_loc(&mut self, file_id: FileId, line: u64, column: u64) -> SourceLoc { + let (index, _) = self.source_loc_set.insert_full((file_id, line, column)); + SourceLoc::new(u32::try_from(index).unwrap()) + } + + pub(super) fn create_debug_lines( + &mut self, + debug_context: &mut DebugContext, + func_id: FuncId, + context: &Context, + ) -> CodeOffset { + let create_row_for_span = + |debug_context: &mut DebugContext, source_loc: (FileId, u64, u64)| { + let (file_id, line, col) = source_loc; + + debug_context.dwarf.unit.line_program.row().file = file_id; + debug_context.dwarf.unit.line_program.row().line = line; + debug_context.dwarf.unit.line_program.row().column = col; + debug_context.dwarf.unit.line_program.generate_row(); + }; + + debug_context.dwarf.unit.line_program.begin_sequence(Some(address_for_func(func_id))); + + let mut func_end = 0; + + let mcr = context.compiled_code().unwrap(); + for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() { + debug_context.dwarf.unit.line_program.row().address_offset = u64::from(start); + if !loc.is_default() { + let source_loc = self.source_loc_set[loc.bits() as usize]; + create_row_for_span(debug_context, source_loc); + } else { + create_row_for_span(debug_context, self.function_source_loc); + } + func_end = end; + } + + debug_context.dwarf.unit.line_program.end_sequence(u64::from(func_end)); + + let func_end = mcr.buffer.total_size(); + + assert_ne!(func_end, 0); + + let entry = debug_context.dwarf.unit.get_mut(self.entry_id); + entry.set(gimli::DW_AT_low_pc, AttributeValue::Address(address_for_func(func_id))); + entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(func_end))); + + func_end + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs new file mode 100644 index 00000000000..f0b78e5d7c6 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs @@ -0,0 +1,337 @@ +//! Handling of everything related to debuginfo. + +mod emit; +mod line_info; +mod object; +mod types; +mod unwind; + +use cranelift_codegen::ir::Endianness; +use cranelift_codegen::isa::TargetIsa; +use cranelift_module::DataId; +use gimli::write::{ + Address, AttributeValue, DwarfUnit, Expression, FileId, LineProgram, LineString, Range, + RangeList, UnitEntryId, +}; +use gimli::{AArch64, Encoding, Format, LineEncoding, Register, RiscV, RunTimeEndian, X86_64}; +use indexmap::IndexSet; +use rustc_codegen_ssa::debuginfo::type_names; +use rustc_hir::def::DefKind; +use rustc_hir::def_id::DefIdMap; +use rustc_session::Session; +use rustc_span::{FileNameDisplayPreference, SourceFileHash, StableSourceFileId}; +use rustc_target::abi::call::FnAbi; + +pub(crate) use self::emit::{DebugReloc, DebugRelocName}; +pub(crate) use self::types::TypeDebugContext; +pub(crate) use self::unwind::UnwindContext; +use crate::debuginfo::emit::{address_for_data, address_for_func}; +use crate::prelude::*; + +pub(crate) fn producer(sess: &Session) -> String { + format!("rustc version {} with cranelift {}", sess.cfg_version, cranelift_codegen::VERSION) +} + +pub(crate) struct DebugContext { + endian: RunTimeEndian, + + dwarf: DwarfUnit, + unit_range_list: RangeList, + created_files: FxHashMap<(StableSourceFileId, SourceFileHash), FileId>, + stack_pointer_register: Register, + namespace_map: DefIdMap<UnitEntryId>, + array_size_type: UnitEntryId, + + filename_display_preference: FileNameDisplayPreference, +} + +pub(crate) struct FunctionDebugContext { + entry_id: UnitEntryId, + function_source_loc: (FileId, u64, u64), + source_loc_set: IndexSet<(FileId, u64, u64)>, +} + +impl DebugContext { + pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, cgu_name: &str) -> Self { + let encoding = Encoding { + format: Format::Dwarf32, + // FIXME this should be configurable + // macOS doesn't seem to support DWARF > 3 + // 5 version is required for md5 file hash + version: if tcx.sess.target.is_like_osx { + 3 + } else { + // FIXME change to version 5 once the gdb and lldb shipping with the latest debian + // support it. + 4 + }, + address_size: isa.frontend_config().pointer_bytes(), + }; + + let endian = match isa.endianness() { + Endianness::Little => RunTimeEndian::Little, + Endianness::Big => RunTimeEndian::Big, + }; + + let stack_pointer_register = match isa.triple().architecture { + target_lexicon::Architecture::Aarch64(_) => AArch64::SP, + target_lexicon::Architecture::Riscv64(_) => RiscV::SP, + target_lexicon::Architecture::X86_64 | target_lexicon::Architecture::X86_64h => { + X86_64::RSP + } + _ => Register(u16::MAX), + }; + + let mut dwarf = DwarfUnit::new(encoding); + + use rustc_session::config::RemapPathScopeComponents; + + let filename_display_preference = + tcx.sess.filename_display_preference(RemapPathScopeComponents::DEBUGINFO); + + let producer = producer(tcx.sess); + let comp_dir = + tcx.sess.opts.working_dir.to_string_lossy(filename_display_preference).to_string(); + + let (name, file_info) = match tcx.sess.local_crate_source_file() { + Some(path) => { + let name = path.to_string_lossy(filename_display_preference).to_string(); + (name, None) + } + None => (tcx.crate_name(LOCAL_CRATE).to_string(), None), + }; + + let mut line_program = LineProgram::new( + encoding, + LineEncoding::default(), + LineString::new(comp_dir.as_bytes(), encoding, &mut dwarf.line_strings), + LineString::new(name.as_bytes(), encoding, &mut dwarf.line_strings), + file_info, + ); + line_program.file_has_md5 = file_info.is_some(); + + dwarf.unit.line_program = line_program; + + { + let name = dwarf.strings.add(format!("{name}/@/{cgu_name}")); + let comp_dir = dwarf.strings.add(comp_dir); + + let root = dwarf.unit.root(); + let root = dwarf.unit.get_mut(root); + root.set(gimli::DW_AT_producer, AttributeValue::StringRef(dwarf.strings.add(producer))); + root.set(gimli::DW_AT_language, AttributeValue::Language(gimli::DW_LANG_Rust)); + root.set(gimli::DW_AT_name, AttributeValue::StringRef(name)); + + // This will be replaced when emitting the debuginfo. It is only + // defined here to ensure that the order of the attributes matches + // rustc. + root.set(gimli::DW_AT_stmt_list, AttributeValue::Udata(0)); + + root.set(gimli::DW_AT_comp_dir, AttributeValue::StringRef(comp_dir)); + root.set(gimli::DW_AT_low_pc, AttributeValue::Address(Address::Constant(0))); + } + + let array_size_type = dwarf.unit.add(dwarf.unit.root(), gimli::DW_TAG_base_type); + let array_size_type_entry = dwarf.unit.get_mut(array_size_type); + array_size_type_entry.set( + gimli::DW_AT_name, + AttributeValue::StringRef(dwarf.strings.add("__ARRAY_SIZE_TYPE__")), + ); + array_size_type_entry + .set(gimli::DW_AT_encoding, AttributeValue::Encoding(gimli::DW_ATE_unsigned)); + array_size_type_entry.set( + gimli::DW_AT_byte_size, + AttributeValue::Udata(isa.frontend_config().pointer_bytes().into()), + ); + + DebugContext { + endian, + dwarf, + unit_range_list: RangeList(Vec::new()), + created_files: FxHashMap::default(), + stack_pointer_register, + namespace_map: DefIdMap::default(), + array_size_type, + filename_display_preference, + } + } + + fn item_namespace(&mut self, tcx: TyCtxt<'_>, def_id: DefId) -> UnitEntryId { + if let Some(&scope) = self.namespace_map.get(&def_id) { + return scope; + } + + let def_key = tcx.def_key(def_id); + let parent_scope = def_key + .parent + .map(|parent| self.item_namespace(tcx, DefId { krate: def_id.krate, index: parent })) + .unwrap_or(self.dwarf.unit.root()); + + let namespace_name = { + let mut output = String::new(); + type_names::push_item_name(tcx, def_id, false, &mut output); + output + }; + let namespace_name_id = self.dwarf.strings.add(namespace_name); + + let scope = self.dwarf.unit.add(parent_scope, gimli::DW_TAG_namespace); + let scope_entry = self.dwarf.unit.get_mut(scope); + scope_entry.set(gimli::DW_AT_name, AttributeValue::StringRef(namespace_name_id)); + + self.namespace_map.insert(def_id, scope); + scope + } + + pub(crate) fn define_function<'tcx>( + &mut self, + tcx: TyCtxt<'tcx>, + type_dbg: &mut TypeDebugContext<'tcx>, + instance: Instance<'tcx>, + fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, + linkage_name: &str, + function_span: Span, + ) -> FunctionDebugContext { + let (file_id, line, column) = self.get_span_loc(tcx, function_span, function_span); + + let scope = self.item_namespace(tcx, tcx.parent(instance.def_id())); + + let mut name = String::new(); + type_names::push_item_name(tcx, instance.def_id(), false, &mut name); + + // Find the enclosing function, in case this is a closure. + let enclosing_fn_def_id = tcx.typeck_root_def_id(instance.def_id()); + + // We look up the generics of the enclosing function and truncate the args + // to their length in order to cut off extra stuff that might be in there for + // closures or coroutines. + let generics = tcx.generics_of(enclosing_fn_def_id); + let args = instance.args.truncate_to(tcx, generics); + + type_names::push_generic_params( + tcx, + tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), args), + enclosing_fn_def_id, + &mut name, + ); + + let entry_id = self.dwarf.unit.add(scope, gimli::DW_TAG_subprogram); + let entry = self.dwarf.unit.get_mut(entry_id); + let linkage_name_id = + if name != linkage_name { Some(self.dwarf.strings.add(linkage_name)) } else { None }; + let name_id = self.dwarf.strings.add(name); + + // These will be replaced in FunctionDebugContext::finalize. They are + // only defined here to ensure that the order of the attributes matches + // rustc. + entry.set(gimli::DW_AT_low_pc, AttributeValue::Udata(0)); + entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(0)); + + let mut frame_base_expr = Expression::new(); + frame_base_expr.op_reg(self.stack_pointer_register); + entry.set(gimli::DW_AT_frame_base, AttributeValue::Exprloc(frame_base_expr)); + + if let Some(linkage_name_id) = linkage_name_id { + entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(linkage_name_id)); + } + // Gdb requires DW_AT_name. Otherwise the DW_TAG_subprogram is skipped. + entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id)); + + entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id))); + entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(line)); + + if !fn_abi.ret.is_ignore() { + let return_dw_ty = self.debug_type(tcx, type_dbg, fn_abi.ret.layout.ty); + let entry = self.dwarf.unit.get_mut(entry_id); + entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(return_dw_ty)); + } + + if tcx.is_reachable_non_generic(instance.def_id()) { + let entry = self.dwarf.unit.get_mut(entry_id); + entry.set(gimli::DW_AT_external, AttributeValue::FlagPresent); + } + + FunctionDebugContext { + entry_id, + function_source_loc: (file_id, line, column), + source_loc_set: IndexSet::new(), + } + } + + // Adapted from https://github.com/rust-lang/rust/blob/10a7aa14fed9b528b74b0f098c4899c37c09a9c7/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs#L1288-L1346 + pub(crate) fn define_static<'tcx>( + &mut self, + tcx: TyCtxt<'tcx>, + type_dbg: &mut TypeDebugContext<'tcx>, + def_id: DefId, + data_id: DataId, + ) { + let DefKind::Static { nested, .. } = tcx.def_kind(def_id) else { bug!() }; + if nested { + return; + } + + let scope = self.item_namespace(tcx, tcx.parent(def_id)); + + let span = tcx.def_span(def_id); + let (file_id, line, _column) = self.get_span_loc(tcx, span, span); + + let static_type = Instance::mono(tcx, def_id).ty(tcx, ty::ParamEnv::reveal_all()); + let static_layout = tcx.layout_of(ty::ParamEnv::reveal_all().and(static_type)).unwrap(); + // FIXME use the actual type layout + let type_id = self.debug_type(tcx, type_dbg, static_type); + + let name = tcx.item_name(def_id); + let linkage_name = tcx.symbol_name(Instance::mono(tcx, def_id)).name; + + let entry_id = self.dwarf.unit.add(scope, gimli::DW_TAG_variable); + let entry = self.dwarf.unit.get_mut(entry_id); + let linkage_name_id = if name.as_str() != linkage_name { + Some(self.dwarf.strings.add(linkage_name)) + } else { + None + }; + let name_id = self.dwarf.strings.add(name.as_str()); + + entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id)); + entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(type_id)); + + if tcx.is_reachable_non_generic(def_id) { + entry.set(gimli::DW_AT_external, AttributeValue::FlagPresent); + } + + entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id))); + entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(line)); + + entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(static_layout.align.pref.bytes())); + + let mut expr = Expression::new(); + expr.op_addr(address_for_data(data_id)); + entry.set(gimli::DW_AT_location, AttributeValue::Exprloc(expr)); + + if let Some(linkage_name_id) = linkage_name_id { + entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(linkage_name_id)); + } + } +} + +impl FunctionDebugContext { + pub(crate) fn finalize( + mut self, + debug_context: &mut DebugContext, + func_id: FuncId, + context: &Context, + ) { + let end = self.create_debug_lines(debug_context, func_id, context); + + debug_context + .unit_range_list + .0 + .push(Range::StartLength { begin: address_for_func(func_id), length: u64::from(end) }); + + let func_entry = debug_context.dwarf.unit.get_mut(self.entry_id); + // Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped. + func_entry.set(gimli::DW_AT_low_pc, AttributeValue::Address(address_for_func(func_id))); + // Using Udata for DW_AT_high_pc requires at least DWARF4 + func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end))); + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs new file mode 100644 index 00000000000..1c6e471cc87 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/object.rs @@ -0,0 +1,91 @@ +use cranelift_module::{DataId, FuncId}; +use cranelift_object::ObjectProduct; +use gimli::SectionId; +use object::write::{Relocation, StandardSegment}; +use object::{RelocationEncoding, RelocationFlags, SectionKind}; +use rustc_data_structures::fx::FxHashMap; + +use crate::debuginfo::{DebugReloc, DebugRelocName}; + +pub(super) trait WriteDebugInfo { + type SectionId: Copy; + + fn add_debug_section(&mut self, name: SectionId, data: Vec<u8>) -> Self::SectionId; + fn add_debug_reloc( + &mut self, + section_map: &FxHashMap<SectionId, Self::SectionId>, + from: &Self::SectionId, + reloc: &DebugReloc, + ); +} + +impl WriteDebugInfo for ObjectProduct { + type SectionId = (object::write::SectionId, object::write::SymbolId); + + fn add_debug_section( + &mut self, + id: SectionId, + data: Vec<u8>, + ) -> (object::write::SectionId, object::write::SymbolId) { + let name = if self.object.format() == object::BinaryFormat::MachO { + id.name().replace('.', "__") // machO expects __debug_info instead of .debug_info + } else { + id.name().to_string() + } + .into_bytes(); + + let segment = self.object.segment_name(StandardSegment::Debug).to_vec(); + // FIXME use SHT_X86_64_UNWIND for .eh_frame + let section_id = self.object.add_section( + segment, + name, + if id == SectionId::DebugStr || id == SectionId::DebugLineStr { + SectionKind::DebugString + } else if id == SectionId::EhFrame { + SectionKind::ReadOnlyData + } else { + SectionKind::Debug + }, + ); + self.object + .section_mut(section_id) + .set_data(data, if id == SectionId::EhFrame { 8 } else { 1 }); + let symbol_id = self.object.section_symbol(section_id); + (section_id, symbol_id) + } + + fn add_debug_reloc( + &mut self, + section_map: &FxHashMap<SectionId, Self::SectionId>, + from: &Self::SectionId, + reloc: &DebugReloc, + ) { + let (symbol, symbol_offset) = match reloc.name { + DebugRelocName::Section(id) => (section_map.get(&id).unwrap().1, 0), + DebugRelocName::Symbol(id) => { + let id = id.try_into().unwrap(); + let symbol_id = if id & 1 << 31 == 0 { + self.function_symbol(FuncId::from_u32(id)) + } else { + self.data_symbol(DataId::from_u32(id & !(1 << 31))) + }; + self.object.symbol_section_and_offset(symbol_id).unwrap_or((symbol_id, 0)) + } + }; + self.object + .add_relocation( + from.0, + Relocation { + offset: u64::from(reloc.offset), + symbol, + flags: RelocationFlags::Generic { + kind: reloc.kind, + encoding: RelocationEncoding::Generic, + size: reloc.size * 8, + }, + addend: i64::try_from(symbol_offset).unwrap() + reloc.addend, + }, + ) + .unwrap(); + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs new file mode 100644 index 00000000000..7baf0a3868d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs @@ -0,0 +1,204 @@ +// Adapted from https://github.com/rust-lang/rust/blob/10a7aa14fed9b528b74b0f098c4899c37c09a9c7/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs + +use gimli::write::{AttributeValue, UnitEntryId}; +use rustc_codegen_ssa::debuginfo::type_names; +use rustc_data_structures::fx::FxHashMap; +use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::{self, Ty, TyCtxt}; + +use crate::{has_ptr_meta, DebugContext, RevealAllLayoutCx}; + +#[derive(Default)] +pub(crate) struct TypeDebugContext<'tcx> { + type_map: FxHashMap<Ty<'tcx>, UnitEntryId>, +} + +/// Returns from the enclosing function if the type debuginfo node with the given +/// unique ID can be found in the type map. +macro_rules! return_if_type_created_in_meantime { + ($type_dbg:expr, $ty:expr) => { + if let Some(&type_id) = $type_dbg.type_map.get(&$ty) { + return type_id; + } + }; +} + +impl DebugContext { + pub(crate) fn debug_type<'tcx>( + &mut self, + tcx: TyCtxt<'tcx>, + type_dbg: &mut TypeDebugContext<'tcx>, + ty: Ty<'tcx>, + ) -> UnitEntryId { + if let Some(&type_id) = type_dbg.type_map.get(&ty) { + return type_id; + } + + let type_id = match ty.kind() { + ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { + self.basic_type(tcx, ty) + } + ty::Tuple(elems) if elems.is_empty() => self.basic_type(tcx, ty), + ty::Array(elem_ty, len) => self.array_type( + tcx, + type_dbg, + ty, + *elem_ty, + len.eval_target_usize(tcx, ty::ParamEnv::reveal_all()), + ), + // ty::Slice(_) | ty::Str + // ty::Dynamic + // ty::Foreign + ty::RawPtr(pointee_type, _) | ty::Ref(_, pointee_type, _) => { + self.pointer_type(tcx, type_dbg, ty, *pointee_type) + } + // ty::Adt(def, args) if def.is_box() && args.get(1).map_or(true, |arg| cx.layout_of(arg.expect_ty()).is_1zst()) + // ty::FnDef(..) | ty::FnPtr(..) + // ty::Closure(..) + // ty::Adt(def, ..) + ty::Tuple(components) => self.tuple_type(tcx, type_dbg, ty, *components), + // ty::Param(_) + // FIXME implement remaining types and add unreachable!() to the fallback branch + _ => self.placeholder_for_type(tcx, type_dbg, ty), + }; + + type_dbg.type_map.insert(ty, type_id); + + type_id + } + + fn basic_type<'tcx>(&mut self, tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> UnitEntryId { + let (name, encoding) = match ty.kind() { + ty::Never => ("!", gimli::DW_ATE_unsigned), + ty::Tuple(elems) if elems.is_empty() => ("()", gimli::DW_ATE_unsigned), + ty::Bool => ("bool", gimli::DW_ATE_boolean), + ty::Char => ("char", gimli::DW_ATE_UTF), + ty::Int(int_ty) => (int_ty.name_str(), gimli::DW_ATE_signed), + ty::Uint(uint_ty) => (uint_ty.name_str(), gimli::DW_ATE_unsigned), + ty::Float(float_ty) => (float_ty.name_str(), gimli::DW_ATE_float), + _ => unreachable!(), + }; + + let type_id = self.dwarf.unit.add(self.dwarf.unit.root(), gimli::DW_TAG_base_type); + let type_entry = self.dwarf.unit.get_mut(type_id); + type_entry.set(gimli::DW_AT_name, AttributeValue::StringRef(self.dwarf.strings.add(name))); + type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(encoding)); + type_entry.set( + gimli::DW_AT_byte_size, + AttributeValue::Udata(RevealAllLayoutCx(tcx).layout_of(ty).size.bytes()), + ); + + type_id + } + + fn array_type<'tcx>( + &mut self, + tcx: TyCtxt<'tcx>, + type_dbg: &mut TypeDebugContext<'tcx>, + array_ty: Ty<'tcx>, + elem_ty: Ty<'tcx>, + len: u64, + ) -> UnitEntryId { + let elem_dw_ty = self.debug_type(tcx, type_dbg, elem_ty); + + return_if_type_created_in_meantime!(type_dbg, array_ty); + + let array_type_id = self.dwarf.unit.add(self.dwarf.unit.root(), gimli::DW_TAG_array_type); + let array_type_entry = self.dwarf.unit.get_mut(array_type_id); + array_type_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(elem_dw_ty)); + + let subrange_id = self.dwarf.unit.add(array_type_id, gimli::DW_TAG_subrange_type); + let subrange_entry = self.dwarf.unit.get_mut(subrange_id); + subrange_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(self.array_size_type)); + subrange_entry.set(gimli::DW_AT_lower_bound, AttributeValue::Udata(0)); + subrange_entry.set(gimli::DW_AT_count, AttributeValue::Udata(len)); + + array_type_id + } + + fn pointer_type<'tcx>( + &mut self, + tcx: TyCtxt<'tcx>, + type_dbg: &mut TypeDebugContext<'tcx>, + ptr_type: Ty<'tcx>, + pointee_type: Ty<'tcx>, + ) -> UnitEntryId { + let pointee_dw_ty = self.debug_type(tcx, type_dbg, pointee_type); + + return_if_type_created_in_meantime!(type_dbg, ptr_type); + + let name = type_names::compute_debuginfo_type_name(tcx, ptr_type, true); + + if !has_ptr_meta(tcx, ptr_type) { + let pointer_type_id = + self.dwarf.unit.add(self.dwarf.unit.root(), gimli::DW_TAG_pointer_type); + let pointer_entry = self.dwarf.unit.get_mut(pointer_type_id); + pointer_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(pointee_dw_ty)); + pointer_entry + .set(gimli::DW_AT_name, AttributeValue::StringRef(self.dwarf.strings.add(name))); + + pointer_type_id + } else { + // FIXME implement debuginfo for fat pointers + self.placeholder_for_type(tcx, type_dbg, ptr_type) + } + } + + fn tuple_type<'tcx>( + &mut self, + tcx: TyCtxt<'tcx>, + type_dbg: &mut TypeDebugContext<'tcx>, + tuple_type: Ty<'tcx>, + components: &'tcx [Ty<'tcx>], + ) -> UnitEntryId { + let components = components + .into_iter() + .map(|&ty| (ty, self.debug_type(tcx, type_dbg, ty))) + .collect::<Vec<_>>(); + + return_if_type_created_in_meantime!(type_dbg, tuple_type); + + let name = type_names::compute_debuginfo_type_name(tcx, tuple_type, false); + let layout = RevealAllLayoutCx(tcx).layout_of(tuple_type); + + let tuple_type_id = + self.dwarf.unit.add(self.dwarf.unit.root(), gimli::DW_TAG_structure_type); + let tuple_entry = self.dwarf.unit.get_mut(tuple_type_id); + tuple_entry.set(gimli::DW_AT_name, AttributeValue::StringRef(self.dwarf.strings.add(name))); + tuple_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes())); + tuple_entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(layout.align.pref.bytes())); + + for (i, (ty, dw_ty)) in components.into_iter().enumerate() { + let member_id = self.dwarf.unit.add(tuple_type_id, gimli::DW_TAG_member); + let member_entry = self.dwarf.unit.get_mut(member_id); + member_entry.set( + gimli::DW_AT_name, + AttributeValue::StringRef(self.dwarf.strings.add(format!("__{i}"))), + ); + member_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty)); + member_entry.set( + gimli::DW_AT_alignment, + AttributeValue::Udata(RevealAllLayoutCx(tcx).layout_of(ty).align.pref.bytes()), + ); + member_entry.set( + gimli::DW_AT_data_member_location, + AttributeValue::Udata(layout.fields.offset(i).bytes()), + ); + } + + tuple_type_id + } + + fn placeholder_for_type<'tcx>( + &mut self, + tcx: TyCtxt<'tcx>, + type_dbg: &mut TypeDebugContext<'tcx>, + ty: Ty<'tcx>, + ) -> UnitEntryId { + self.debug_type( + tcx, + type_dbg, + Ty::new_array(tcx, tcx.types.u8, RevealAllLayoutCx(tcx).layout_of(ty).size.bytes()), + ) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs new file mode 100644 index 00000000000..ac7dd0bd463 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs @@ -0,0 +1,143 @@ +//! Unwind info generation (`.eh_frame`) + +use cranelift_codegen::ir::Endianness; +use cranelift_codegen::isa::unwind::UnwindInfo; +use cranelift_codegen::isa::TargetIsa; +use cranelift_object::ObjectProduct; +use gimli::write::{CieId, EhFrame, FrameTable, Section}; +use gimli::RunTimeEndian; + +use super::emit::address_for_func; +use super::object::WriteDebugInfo; +use crate::prelude::*; + +pub(crate) struct UnwindContext { + endian: RunTimeEndian, + frame_table: FrameTable, + cie_id: Option<CieId>, +} + +impl UnwindContext { + pub(crate) fn new(isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { + let endian = match isa.endianness() { + Endianness::Little => RunTimeEndian::Little, + Endianness::Big => RunTimeEndian::Big, + }; + let mut frame_table = FrameTable::default(); + + let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { + if pic_eh_frame { + cie.fde_address_encoding = + gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0); + } + Some(frame_table.add_cie(cie)) + } else { + None + }; + + UnwindContext { endian, frame_table, cie_id } + } + + pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) { + if let target_lexicon::OperatingSystem::MacOSX { .. } = isa.triple().operating_system { + // The object crate doesn't currently support DW_GNU_EH_PE_absptr, which macOS + // requires for unwinding tables. In addition on arm64 it currently doesn't + // support 32bit relocations as we currently use for the unwinding table. + // See gimli-rs/object#415 and rust-lang/rustc_codegen_cranelift#1371 + return; + } + + let unwind_info = if let Some(unwind_info) = + context.compiled_code().unwrap().create_unwind_info(isa).unwrap() + { + unwind_info + } else { + return; + }; + + match unwind_info { + UnwindInfo::SystemV(unwind_info) => { + self.frame_table + .add_fde(self.cie_id.unwrap(), unwind_info.to_fde(address_for_func(func_id))); + } + UnwindInfo::WindowsX64(_) => { + // FIXME implement this + } + unwind_info => unimplemented!("{:?}", unwind_info), + } + } + + pub(crate) fn emit(self, product: &mut ObjectProduct) { + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); + self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); + + if !eh_frame.0.writer.slice().is_empty() { + let id = eh_frame.id(); + let section_id = product.add_debug_section(id, eh_frame.0.writer.into_vec()); + let mut section_map = FxHashMap::default(); + section_map.insert(id, section_id); + + for reloc in &eh_frame.0.relocs { + product.add_debug_reloc(§ion_map, §ion_id, reloc); + } + } + } + + #[cfg(all(feature = "jit", windows))] + pub(crate) unsafe fn register_jit(self, _jit_module: &cranelift_jit::JITModule) {} + + #[cfg(all(feature = "jit", not(windows)))] + pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) { + use std::mem::ManuallyDrop; + + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); + self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); + + if eh_frame.0.writer.slice().is_empty() { + return; + } + + let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module); + + // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. + eh_frame.extend(&[0, 0, 0, 0]); + + // FIXME support unregistering unwind tables once cranelift-jit supports deallocating + // individual functions + let eh_frame = ManuallyDrop::new(eh_frame); + + // ======================================================================= + // Everything after this line up to the end of the file is loosely based on + // https://github.com/bytecodealliance/wasmtime/blob/4471a82b0c540ff48960eca6757ccce5b1b5c3e4/crates/jit/src/unwind/systemv.rs + #[cfg(target_os = "macos")] + { + // On macOS, `__register_frame` takes a pointer to a single FDE + let start = eh_frame.as_ptr(); + let end = start.add(eh_frame.len()); + let mut current = start; + + // Walk all of the entries in the frame table and register them + while current < end { + let len = std::ptr::read::<u32>(current as *const u32) as usize; + + // Skip over the CIE + if current != start { + __register_frame(current); + } + + // Move to the next table entry (+4 because the length itself is not inclusive) + current = current.add(len + 4); + } + } + #[cfg(not(target_os = "macos"))] + { + // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0 + __register_frame(eh_frame.as_ptr()); + } + } +} + +extern "C" { + // libunwind import + fn __register_frame(fde: *const u8); +} diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs new file mode 100644 index 00000000000..e7ac084558a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs @@ -0,0 +1,214 @@ +//! Handling of enum discriminants +//! +//! Adapted from <https://github.com/rust-lang/rust/blob/31c0645b9d2539f47eecb096142474b29dc542f7/compiler/rustc_codegen_ssa/src/mir/place.rs> +//! (<https://github.com/rust-lang/rust/pull/104535>) + +use rustc_target::abi::{Int, TagEncoding, Variants}; + +use crate::prelude::*; + +pub(crate) fn codegen_set_discriminant<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + place: CPlace<'tcx>, + variant_index: VariantIdx, +) { + let layout = place.layout(); + if layout.for_variant(fx, variant_index).abi.is_uninhabited() { + return; + } + match layout.variants { + Variants::Single { index } => { + assert_eq!(index, variant_index); + } + Variants::Multiple { + tag: _, + tag_field, + tag_encoding: TagEncoding::Direct, + variants: _, + } => { + let ptr = place.place_field(fx, FieldIdx::new(tag_field)); + let to = layout.ty.discriminant_for_variant(fx.tcx, variant_index).unwrap().val; + let to = match ptr.layout().ty.kind() { + ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { + let lsb = fx.bcx.ins().iconst(types::I64, to as u64 as i64); + let msb = fx.bcx.ins().iconst(types::I64, (to >> 64) as u64 as i64); + fx.bcx.ins().iconcat(lsb, msb) + } + ty::Uint(_) | ty::Int(_) => { + let clif_ty = fx.clif_type(ptr.layout().ty).unwrap(); + let raw_val = ptr.layout().size.truncate(to); + fx.bcx.ins().iconst(clif_ty, raw_val as i64) + } + _ => unreachable!(), + }; + let discr = CValue::by_val(to, ptr.layout()); + ptr.write_cvalue(fx, discr); + } + Variants::Multiple { + tag: _, + tag_field, + tag_encoding: TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, + variants: _, + } => { + if variant_index != untagged_variant { + let niche = place.place_field(fx, FieldIdx::new(tag_field)); + let niche_type = fx.clif_type(niche.layout().ty).unwrap(); + let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); + let niche_value = (niche_value as u128).wrapping_add(niche_start); + let niche_value = match niche_type { + types::I128 => { + let lsb = fx.bcx.ins().iconst(types::I64, niche_value as u64 as i64); + let msb = + fx.bcx.ins().iconst(types::I64, (niche_value >> 64) as u64 as i64); + fx.bcx.ins().iconcat(lsb, msb) + } + ty => fx.bcx.ins().iconst(ty, niche_value as i64), + }; + let niche_llval = CValue::by_val(niche_value, niche.layout()); + niche.write_cvalue(fx, niche_llval); + } + } + } +} + +pub(crate) fn codegen_get_discriminant<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + dest: CPlace<'tcx>, + value: CValue<'tcx>, + dest_layout: TyAndLayout<'tcx>, +) { + let layout = value.layout(); + + if layout.abi.is_uninhabited() { + return; + } + + let (tag_scalar, tag_field, tag_encoding) = match &layout.variants { + Variants::Single { index } => { + let discr_val = layout + .ty + .discriminant_for_variant(fx.tcx, *index) + .map_or(u128::from(index.as_u32()), |discr| discr.val); + + let val = match dest_layout.ty.kind() { + ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { + let lsb = fx.bcx.ins().iconst(types::I64, discr_val as u64 as i64); + let msb = fx.bcx.ins().iconst(types::I64, (discr_val >> 64) as u64 as i64); + fx.bcx.ins().iconcat(lsb, msb) + } + ty::Uint(_) | ty::Int(_) => { + let clif_ty = fx.clif_type(dest_layout.ty).unwrap(); + let raw_val = dest_layout.size.truncate(discr_val); + fx.bcx.ins().iconst(clif_ty, raw_val as i64) + } + _ => unreachable!(), + }; + let res = CValue::by_val(val, dest_layout); + dest.write_cvalue(fx, res); + return; + } + Variants::Multiple { tag, tag_field, tag_encoding, variants: _ } => { + (tag, *tag_field, tag_encoding) + } + }; + + let cast_to = fx.clif_type(dest_layout.ty).unwrap(); + + // Read the tag/niche-encoded discriminant from memory. + let tag = value.value_field(fx, FieldIdx::new(tag_field)); + let tag = tag.load_scalar(fx); + + // Decode the discriminant (specifically if it's niche-encoded). + match *tag_encoding { + TagEncoding::Direct => { + let signed = match tag_scalar.primitive() { + Int(_, signed) => signed, + _ => false, + }; + let val = clif_intcast(fx, tag, cast_to, signed); + let res = CValue::by_val(val, dest_layout); + dest.write_cvalue(fx, res); + } + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => { + let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32(); + + // We have a subrange `niche_start..=niche_end` inside `range`. + // If the value of the tag is inside this subrange, it's a + // "niche value", an increment of the discriminant. Otherwise it + // indicates the untagged variant. + // A general algorithm to extract the discriminant from the tag + // is: + // relative_tag = tag - niche_start + // is_niche = relative_tag <= (ule) relative_max + // discr = if is_niche { + // cast(relative_tag) + niche_variants.start() + // } else { + // untagged_variant + // } + // However, we will likely be able to emit simpler code. + + let (is_niche, tagged_discr, delta) = if relative_max == 0 { + // Best case scenario: only one tagged variant. This will + // likely become just a comparison and a jump. + // The algorithm is: + // is_niche = tag == niche_start + // discr = if is_niche { + // niche_start + // } else { + // untagged_variant + // } + let is_niche = codegen_icmp_imm(fx, IntCC::Equal, tag, niche_start as i128); + let tagged_discr = + fx.bcx.ins().iconst(cast_to, niche_variants.start().as_u32() as i64); + (is_niche, tagged_discr, 0) + } else { + // The special cases don't apply, so we'll have to go with + // the general algorithm. + let niche_start = match fx.bcx.func.dfg.value_type(tag) { + types::I128 => { + let lsb = fx.bcx.ins().iconst(types::I64, niche_start as u64 as i64); + let msb = + fx.bcx.ins().iconst(types::I64, (niche_start >> 64) as u64 as i64); + fx.bcx.ins().iconcat(lsb, msb) + } + ty => fx.bcx.ins().iconst(ty, niche_start as i64), + }; + let relative_discr = fx.bcx.ins().isub(tag, niche_start); + let cast_tag = clif_intcast(fx, relative_discr, cast_to, false); + let is_niche = crate::common::codegen_icmp_imm( + fx, + IntCC::UnsignedLessThanOrEqual, + relative_discr, + i128::from(relative_max), + ); + (is_niche, cast_tag, niche_variants.start().as_u32() as u128) + }; + + let tagged_discr = if delta == 0 { + tagged_discr + } else { + let delta = match cast_to { + types::I128 => { + let lsb = fx.bcx.ins().iconst(types::I64, delta as u64 as i64); + let msb = fx.bcx.ins().iconst(types::I64, (delta >> 64) as u64 as i64); + fx.bcx.ins().iconcat(lsb, msb) + } + ty => fx.bcx.ins().iconst(ty, delta as i64), + }; + fx.bcx.ins().iadd(tagged_discr, delta) + }; + + let untagged_variant = if cast_to == types::I128 { + let zero = fx.bcx.ins().iconst(types::I64, 0); + let untagged_variant = + fx.bcx.ins().iconst(types::I64, i64::from(untagged_variant.as_u32())); + fx.bcx.ins().iconcat(untagged_variant, zero) + } else { + fx.bcx.ins().iconst(cast_to, i64::from(untagged_variant.as_u32())) + }; + let discr = fx.bcx.ins().select(is_niche, tagged_discr, untagged_variant); + let res = CValue::by_val(discr, dest_layout); + dest.write_cvalue(fx, res); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs new file mode 100644 index 00000000000..b6fee1bf24a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -0,0 +1,728 @@ +//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a +//! standalone executable. + +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::thread::JoinHandle; + +use cranelift_object::{ObjectBuilder, ObjectModule}; +use rustc_codegen_ssa::assert_module_sources::CguReuse; +use rustc_codegen_ssa::back::link::ensure_removed; +use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file; +use rustc_codegen_ssa::base::determine_cgu_reuse; +use rustc_codegen_ssa::{ + errors as ssa_errors, CodegenResults, CompiledModule, CrateInfo, ModuleKind, +}; +use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_data_structures::sync::{par_map, IntoDynSyncSend}; +use rustc_metadata::fs::copy_to_stdout; +use rustc_metadata::EncodedMetadata; +use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; +use rustc_session::config::{DebugInfo, OutFileName, OutputFilenames, OutputType}; +use rustc_session::Session; + +use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken}; +use crate::debuginfo::TypeDebugContext; +use crate::global_asm::GlobalAsmConfig; +use crate::prelude::*; +use crate::unwind_module::UnwindModule; +use crate::BackendConfig; + +struct ModuleCodegenResult { + module_regular: CompiledModule, + module_global_asm: Option<CompiledModule>, + existing_work_product: Option<(WorkProductId, WorkProduct)>, +} + +enum OngoingModuleCodegen { + Sync(Result<ModuleCodegenResult, String>), + Async(JoinHandle<Result<ModuleCodegenResult, String>>), +} + +impl<HCX> HashStable<HCX> for OngoingModuleCodegen { + fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { + // do nothing + } +} + +pub(crate) struct OngoingCodegen { + modules: Vec<OngoingModuleCodegen>, + allocator_module: Option<CompiledModule>, + metadata_module: Option<CompiledModule>, + metadata: EncodedMetadata, + crate_info: CrateInfo, + concurrency_limiter: ConcurrencyLimiter, +} + +impl OngoingCodegen { + pub(crate) fn join( + self, + sess: &Session, + outputs: &OutputFilenames, + backend_config: &BackendConfig, + ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) { + let mut work_products = FxIndexMap::default(); + let mut modules = vec![]; + + for module_codegen in self.modules { + let module_codegen_result = match module_codegen { + OngoingModuleCodegen::Sync(module_codegen_result) => module_codegen_result, + OngoingModuleCodegen::Async(join_handle) => match join_handle.join() { + Ok(module_codegen_result) => module_codegen_result, + Err(panic) => std::panic::resume_unwind(panic), + }, + }; + + let module_codegen_result = match module_codegen_result { + Ok(module_codegen_result) => module_codegen_result, + Err(err) => sess.dcx().fatal(err), + }; + let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } = + module_codegen_result; + + if let Some((work_product_id, work_product)) = existing_work_product { + work_products.insert(work_product_id, work_product); + } else { + let work_product = if backend_config.disable_incr_cache { + None + } else if let Some(module_global_asm) = &module_global_asm { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + sess, + &module_regular.name, + &[ + ("o", &module_regular.object.as_ref().unwrap()), + ("asm.o", &module_global_asm.object.as_ref().unwrap()), + ], + ) + } else { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + sess, + &module_regular.name, + &[("o", &module_regular.object.as_ref().unwrap())], + ) + }; + if let Some((work_product_id, work_product)) = work_product { + work_products.insert(work_product_id, work_product); + } + } + + modules.push(module_regular); + if let Some(module_global_asm) = module_global_asm { + modules.push(module_global_asm); + } + } + + self.concurrency_limiter.finished(); + + sess.dcx().abort_if_errors(); + + let codegen_results = CodegenResults { + modules, + allocator_module: self.allocator_module, + metadata_module: self.metadata_module, + metadata: self.metadata, + crate_info: self.crate_info, + }; + + produce_final_output_artifacts(sess, &codegen_results, outputs); + + (codegen_results, work_products) + } +} + +// Adapted from https://github.com/rust-lang/rust/blob/73476d49904751f8d90ce904e16dfbc278083d2c/compiler/rustc_codegen_ssa/src/back/write.rs#L547C1-L706C2 +fn produce_final_output_artifacts( + sess: &Session, + codegen_results: &CodegenResults, + crate_output: &OutputFilenames, +) { + let user_wants_bitcode = false; + let mut user_wants_objects = false; + + // Produce final compile outputs. + let copy_gracefully = |from: &Path, to: &OutFileName| match to { + OutFileName::Stdout => { + if let Err(e) = copy_to_stdout(from) { + sess.dcx().emit_err(ssa_errors::CopyPath::new(from, to.as_path(), e)); + } + } + OutFileName::Real(path) => { + if let Err(e) = fs::copy(from, path) { + sess.dcx().emit_err(ssa_errors::CopyPath::new(from, path, e)); + } + } + }; + + let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| { + if codegen_results.modules.len() == 1 { + // 1) Only one codegen unit. In this case it's no difficulty + // to copy `foo.0.x` to `foo.x`. + let module_name = Some(&codegen_results.modules[0].name[..]); + let path = crate_output.temp_path(output_type, module_name); + let output = crate_output.path(output_type); + if !output_type.is_text_output() && output.is_tty() { + sess.dcx() + .emit_err(ssa_errors::BinaryOutputToTty { shorthand: output_type.shorthand() }); + } else { + copy_gracefully(&path, &output); + } + if !sess.opts.cg.save_temps && !keep_numbered { + // The user just wants `foo.x`, not `foo.#module-name#.x`. + ensure_removed(sess.dcx(), &path); + } + } else { + let extension = crate_output + .temp_path(output_type, None) + .extension() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + + if crate_output.outputs.contains_explicit_name(&output_type) { + // 2) Multiple codegen units, with `--emit foo=some_name`. We have + // no good solution for this case, so warn the user. + sess.dcx().emit_warn(ssa_errors::IgnoringEmitPath { extension }); + } else if crate_output.single_output_file.is_some() { + // 3) Multiple codegen units, with `-o some_name`. We have + // no good solution for this case, so warn the user. + sess.dcx().emit_warn(ssa_errors::IgnoringOutput { extension }); + } else { + // 4) Multiple codegen units, but no explicit name. We + // just leave the `foo.0.x` files in place. + // (We don't have to do any work in this case.) + } + } + }; + + // Flag to indicate whether the user explicitly requested bitcode. + // Otherwise, we produced it only as a temporary output, and will need + // to get rid of it. + for output_type in crate_output.outputs.keys() { + match *output_type { + OutputType::Bitcode | OutputType::ThinLinkBitcode => { + // Cranelift doesn't have bitcode + // user_wants_bitcode = true; + // // Copy to .bc, but always keep the .0.bc. There is a later + // // check to figure out if we should delete .0.bc files, or keep + // // them for making an rlib. + // copy_if_one_unit(OutputType::Bitcode, true); + } + OutputType::LlvmAssembly => { + // Cranelift IR text already emitted during codegen + // copy_if_one_unit(OutputType::LlvmAssembly, false); + } + OutputType::Assembly => { + // Currently no support for emitting raw assembly files + // copy_if_one_unit(OutputType::Assembly, false); + } + OutputType::Object => { + user_wants_objects = true; + copy_if_one_unit(OutputType::Object, true); + } + OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {} + } + } + + // Clean up unwanted temporary files. + + // We create the following files by default: + // - #crate#.#module-name#.bc + // - #crate#.#module-name#.o + // - #crate#.crate.metadata.bc + // - #crate#.crate.metadata.o + // - #crate#.o (linked from crate.##.o) + // - #crate#.bc (copied from crate.##.bc) + // We may create additional files if requested by the user (through + // `-C save-temps` or `--emit=` flags). + + if !sess.opts.cg.save_temps { + // Remove the temporary .#module-name#.o objects. If the user didn't + // explicitly request bitcode (with --emit=bc), and the bitcode is not + // needed for building an rlib, then we must remove .#module-name#.bc as + // well. + + // Specific rules for keeping .#module-name#.bc: + // - If the user requested bitcode (`user_wants_bitcode`), and + // codegen_units > 1, then keep it. + // - If the user requested bitcode but codegen_units == 1, then we + // can toss .#module-name#.bc because we copied it to .bc earlier. + // - If we're not building an rlib and the user didn't request + // bitcode, then delete .#module-name#.bc. + // If you change how this works, also update back::link::link_rlib, + // where .#module-name#.bc files are (maybe) deleted after making an + // rlib. + let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe); + + let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1; + + let keep_numbered_objects = + needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1); + + for module in codegen_results.modules.iter() { + if let Some(ref path) = module.object { + if !keep_numbered_objects { + ensure_removed(sess.dcx(), path); + } + } + + if let Some(ref path) = module.dwarf_object { + if !keep_numbered_objects { + ensure_removed(sess.dcx(), path); + } + } + + if let Some(ref path) = module.bytecode { + if !keep_numbered_bitcode { + ensure_removed(sess.dcx(), path); + } + } + } + + if !user_wants_bitcode { + if let Some(ref allocator_module) = codegen_results.allocator_module { + if let Some(ref path) = allocator_module.bytecode { + ensure_removed(sess.dcx(), path); + } + } + } + } + + if sess.opts.json_artifact_notifications { + if codegen_results.modules.len() == 1 { + codegen_results.modules[0].for_each_output(|_path, ty| { + if sess.opts.output_types.contains_key(&ty) { + let descr = ty.shorthand(); + // for single cgu file is renamed to drop cgu specific suffix + // so we regenerate it the same way + let path = crate_output.path(ty); + sess.dcx().emit_artifact_notification(path.as_path(), descr); + } + }); + } else { + for module in &codegen_results.modules { + module.for_each_output(|path, ty| { + if sess.opts.output_types.contains_key(&ty) { + let descr = ty.shorthand(); + sess.dcx().emit_artifact_notification(&path, descr); + } + }); + } + } + } + + // We leave the following files around by default: + // - #crate#.o + // - #crate#.crate.metadata.o + // - #crate#.bc + // These are used in linking steps and will be cleaned up afterward. +} + +fn make_module( + sess: &Session, + backend_config: &BackendConfig, + name: String, +) -> UnwindModule<ObjectModule> { + let isa = crate::build_isa(sess, backend_config); + + let mut builder = + ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); + // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size + // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections + // can easily double the amount of time necessary to perform linking. + builder.per_function_section(sess.opts.unstable_opts.function_sections.unwrap_or(false)); + UnwindModule::new(ObjectModule::new(builder), true) +} + +fn emit_cgu( + output_filenames: &OutputFilenames, + prof: &SelfProfilerRef, + name: String, + module: UnwindModule<ObjectModule>, + debug: Option<DebugContext>, + global_asm_object_file: Option<PathBuf>, + producer: &str, +) -> Result<ModuleCodegenResult, String> { + let mut product = module.finish(); + + if let Some(mut debug) = debug { + debug.emit(&mut product); + } + + let module_regular = emit_module( + output_filenames, + prof, + product.object, + ModuleKind::Regular, + name.clone(), + producer, + )?; + + Ok(ModuleCodegenResult { + module_regular, + module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule { + name: format!("{name}.asm"), + kind: ModuleKind::Regular, + object: Some(global_asm_object_file), + dwarf_object: None, + bytecode: None, + assembly: None, + llvm_ir: None, + }), + existing_work_product: None, + }) +} + +fn emit_module( + output_filenames: &OutputFilenames, + prof: &SelfProfilerRef, + mut object: cranelift_object::object::write::Object<'_>, + kind: ModuleKind, + name: String, + producer_str: &str, +) -> Result<CompiledModule, String> { + if object.format() == cranelift_object::object::BinaryFormat::Elf { + let comment_section = object.add_section( + Vec::new(), + b".comment".to_vec(), + cranelift_object::object::SectionKind::OtherString, + ); + let mut producer = vec![0]; + producer.extend(producer_str.as_bytes()); + producer.push(0); + object.set_section_data(comment_section, producer, 1); + } + + let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name)); + let mut file = match File::create(&tmp_file) { + Ok(file) => file, + Err(err) => return Err(format!("error creating object file: {}", err)), + }; + + if let Err(err) = object.write_stream(&mut file) { + return Err(format!("error writing object file: {}", err)); + } + + prof.artifact_size("object_file", &*name, file.metadata().unwrap().len()); + + Ok(CompiledModule { + name, + kind, + object: Some(tmp_file), + dwarf_object: None, + bytecode: None, + assembly: None, + llvm_ir: None, + }) +} + +fn reuse_workproduct_for_cgu( + tcx: TyCtxt<'_>, + cgu: &CodegenUnit<'_>, +) -> Result<ModuleCodegenResult, String> { + let work_product = cgu.previous_work_product(tcx); + let obj_out_regular = + tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); + let source_file_regular = rustc_incremental::in_incr_comp_dir_sess( + &tcx.sess, + &work_product.saved_files.get("o").expect("no saved object file in work product"), + ); + + if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) { + return Err(format!( + "unable to copy {} to {}: {}", + source_file_regular.display(), + obj_out_regular.display(), + err + )); + } + let obj_out_global_asm = + crate::global_asm::add_file_stem_postfix(obj_out_regular.clone(), ".asm"); + let has_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") { + let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, asm_o); + if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm) + { + return Err(format!( + "unable to copy {} to {}: {}", + source_file_regular.display(), + obj_out_regular.display(), + err + )); + } + true + } else { + false + }; + + Ok(ModuleCodegenResult { + module_regular: CompiledModule { + name: cgu.name().to_string(), + kind: ModuleKind::Regular, + object: Some(obj_out_regular), + dwarf_object: None, + bytecode: None, + assembly: None, + llvm_ir: None, + }, + module_global_asm: has_global_asm.then(|| CompiledModule { + name: cgu.name().to_string(), + kind: ModuleKind::Regular, + object: Some(obj_out_global_asm), + dwarf_object: None, + bytecode: None, + assembly: None, + llvm_ir: None, + }), + existing_work_product: Some((cgu.work_product_id(), work_product)), + }) +} + +fn module_codegen( + tcx: TyCtxt<'_>, + (backend_config, global_asm_config, cgu_name, token): ( + BackendConfig, + Arc<GlobalAsmConfig>, + rustc_span::Symbol, + ConcurrencyLimiterToken, + ), +) -> OngoingModuleCodegen { + let (cgu_name, mut cx, mut module, codegened_functions) = + tcx.prof.generic_activity_with_arg("codegen cgu", cgu_name.as_str()).run(|| { + let cgu = tcx.codegen_unit(cgu_name); + let mono_items = cgu.items_in_deterministic_order(tcx); + + let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string()); + + let mut cx = crate::CodegenCx::new( + tcx, + module.isa(), + tcx.sess.opts.debuginfo != DebugInfo::None, + cgu_name, + ); + let mut type_dbg = TypeDebugContext::default(); + super::predefine_mono_items(tcx, &mut module, &mono_items); + let mut codegened_functions = vec![]; + for (mono_item, _) in mono_items { + match mono_item { + MonoItem::Fn(inst) => { + if let Some(codegened_function) = crate::base::codegen_fn( + tcx, + &mut cx, + &mut type_dbg, + Function::new(), + &mut module, + inst, + ) { + codegened_functions.push(codegened_function); + } + } + MonoItem::Static(def_id) => { + let data_id = crate::constant::codegen_static(tcx, &mut module, def_id); + if let Some(debug_context) = &mut cx.debug_context { + debug_context.define_static(tcx, &mut type_dbg, def_id, data_id); + } + } + MonoItem::GlobalAsm(item_id) => { + crate::global_asm::codegen_global_asm_item( + tcx, + &mut cx.global_asm, + item_id, + ); + } + } + } + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, false, cgu.is_primary()); + + let cgu_name = cgu.name().as_str().to_owned(); + + (cgu_name, cx, module, codegened_functions) + }); + + let producer = crate::debuginfo::producer(tcx.sess); + + OngoingModuleCodegen::Async(std::thread::spawn(move || { + cx.profiler.clone().generic_activity_with_arg("compile functions", &*cgu_name).run(|| { + cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( + cx.profiler.clone(), + ))); + + let mut cached_context = Context::new(); + for codegened_func in codegened_functions { + crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func); + } + }); + + let global_asm_object_file = + cx.profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| { + crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm) + })?; + + let codegen_result = + cx.profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| { + emit_cgu( + &global_asm_config.output_filenames, + &cx.profiler, + cgu_name, + module, + cx.debug_context, + global_asm_object_file, + &producer, + ) + }); + std::mem::drop(token); + codegen_result + })) +} + +pub(crate) fn run_aot( + tcx: TyCtxt<'_>, + backend_config: BackendConfig, + metadata: EncodedMetadata, + need_metadata_module: bool, +) -> Box<OngoingCodegen> { + // FIXME handle `-Ctarget-cpu=native` + let target_cpu = match tcx.sess.opts.cg.target_cpu { + Some(ref name) => name, + None => tcx.sess.target.cpu.as_ref(), + } + .to_owned(); + + let cgus = if tcx.sess.opts.output_types.should_codegen() { + tcx.collect_and_partition_mono_items(()).1 + } else { + // If only `--emit metadata` is used, we shouldn't perform any codegen. + // Also `tcx.collect_and_partition_mono_items` may panic in that case. + return Box::new(OngoingCodegen { + modules: vec![], + allocator_module: None, + metadata_module: None, + metadata, + crate_info: CrateInfo::new(tcx, target_cpu), + concurrency_limiter: ConcurrencyLimiter::new(tcx.sess, 0), + }); + }; + + if tcx.dep_graph.is_fully_enabled() { + for cgu in cgus { + tcx.ensure().codegen_unit(cgu.name()); + } + } + + // Calculate the CGU reuse + let cgu_reuse = tcx.sess.time("find_cgu_reuse", || { + cgus.iter().map(|cgu| determine_cgu_reuse(tcx, &cgu)).collect::<Vec<_>>() + }); + + rustc_codegen_ssa::assert_module_sources::assert_module_sources(tcx, &|cgu_reuse_tracker| { + for (i, cgu) in cgus.iter().enumerate() { + let cgu_reuse = cgu_reuse[i]; + cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); + } + }); + + let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx)); + + let (todo_cgus, done_cgus) = + cgus.into_iter().enumerate().partition::<Vec<_>, _>(|&(i, _)| match cgu_reuse[i] { + _ if backend_config.disable_incr_cache => true, + CguReuse::No => true, + CguReuse::PreLto | CguReuse::PostLto => false, + }); + + let concurrency_limiter = IntoDynSyncSend(ConcurrencyLimiter::new(tcx.sess, todo_cgus.len())); + + let modules = tcx.sess.time("codegen mono items", || { + let mut modules: Vec<_> = par_map(todo_cgus, |(_, cgu)| { + let dep_node = cgu.codegen_dep_node(tcx); + tcx.dep_graph + .with_task( + dep_node, + tcx, + ( + backend_config.clone(), + global_asm_config.clone(), + cgu.name(), + concurrency_limiter.acquire(tcx.dcx()), + ), + module_codegen, + Some(rustc_middle::dep_graph::hash_result), + ) + .0 + }); + modules.extend( + done_cgus + .into_iter() + .map(|(_, cgu)| OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu))), + ); + modules + }); + + let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string()); + let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module); + + let allocator_module = if created_alloc_shim { + let product = allocator_module.finish(); + + match emit_module( + tcx.output_filenames(()), + &tcx.sess.prof, + product.object, + ModuleKind::Allocator, + "allocator_shim".to_owned(), + &crate::debuginfo::producer(tcx.sess), + ) { + Ok(allocator_module) => Some(allocator_module), + Err(err) => tcx.dcx().fatal(err), + } + } else { + None + }; + + let metadata_module = if need_metadata_module { + let (metadata_cgu_name, tmp_file) = tcx.sess.time("write compressed metadata", || { + use rustc_middle::mir::mono::CodegenUnitNameBuilder; + + let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx); + let metadata_cgu_name = cgu_name_builder + .build_cgu_name(LOCAL_CRATE, ["crate"], Some("metadata")) + .as_str() + .to_string(); + + let tmp_file = + tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); + + let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx); + let obj = create_compressed_metadata_file(tcx.sess, &metadata, &symbol_name); + + if let Err(err) = std::fs::write(&tmp_file, obj) { + tcx.dcx().fatal(format!("error writing metadata object file: {}", err)); + } + + (metadata_cgu_name, tmp_file) + }); + + Some(CompiledModule { + name: metadata_cgu_name, + kind: ModuleKind::Metadata, + object: Some(tmp_file), + dwarf_object: None, + bytecode: None, + assembly: None, + llvm_ir: None, + }) + } else { + None + }; + + Box::new(OngoingCodegen { + modules, + allocator_module, + metadata_module, + metadata, + crate_info: CrateInfo::new(tcx, target_cpu), + concurrency_limiter: concurrency_limiter.0, + }) +} diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs new file mode 100644 index 00000000000..12e860f676d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -0,0 +1,399 @@ +//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object +//! files. + +use std::cell::RefCell; +use std::ffi::CString; +use std::os::raw::{c_char, c_int}; +use std::sync::{mpsc, Mutex, OnceLock}; + +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use cranelift_jit::{JITBuilder, JITModule}; +use rustc_codegen_ssa::CrateInfo; +use rustc_middle::mir::mono::MonoItem; +use rustc_session::Session; +use rustc_span::Symbol; + +use crate::debuginfo::TypeDebugContext; +use crate::prelude::*; +use crate::unwind_module::UnwindModule; +use crate::{BackendConfig, CodegenCx, CodegenMode}; + +struct JitState { + jit_module: UnwindModule<JITModule>, +} + +thread_local! { + static LAZY_JIT_STATE: RefCell<Option<JitState>> = const { RefCell::new(None) }; +} + +/// The Sender owned by the rustc thread +static GLOBAL_MESSAGE_SENDER: OnceLock<Mutex<mpsc::Sender<UnsafeMessage>>> = OnceLock::new(); + +/// A message that is sent from the jitted runtime to the rustc thread. +/// Senders are responsible for upholding `Send` semantics. +enum UnsafeMessage { + /// Request that the specified `Instance` be lazily jitted. + /// + /// Nothing accessible through `instance_ptr` may be moved or mutated by the sender after + /// this message is sent. + JitFn { + instance_ptr: *const Instance<'static>, + trampoline_ptr: *const u8, + tx: mpsc::Sender<*const u8>, + }, +} +unsafe impl Send for UnsafeMessage {} + +impl UnsafeMessage { + /// Send the message. + fn send(self) -> Result<(), mpsc::SendError<UnsafeMessage>> { + thread_local! { + /// The Sender owned by the local thread + static LOCAL_MESSAGE_SENDER: mpsc::Sender<UnsafeMessage> = + GLOBAL_MESSAGE_SENDER + .get().unwrap() + .lock().unwrap() + .clone(); + } + LOCAL_MESSAGE_SENDER.with(|sender| sender.send(self)) + } +} + +fn create_jit_module( + tcx: TyCtxt<'_>, + backend_config: &BackendConfig, + hotswap: bool, +) -> (UnwindModule<JITModule>, CodegenCx) { + let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string()); + + let isa = crate::build_isa(tcx.sess, backend_config); + let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); + jit_builder.hotswap(hotswap); + crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); + jit_builder.symbol_lookup_fn(dep_symbol_lookup_fn(tcx.sess, crate_info)); + jit_builder.symbol("__clif_jit_fn", clif_jit_fn as *const u8); + let mut jit_module = UnwindModule::new(JITModule::new(jit_builder), false); + + let cx = crate::CodegenCx::new(tcx, jit_module.isa(), false, Symbol::intern("dummy_cgu_name")); + + crate::allocator::codegen(tcx, &mut jit_module); + + (jit_module, cx) +} + +pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { + if !tcx.sess.opts.output_types.should_codegen() { + tcx.dcx().fatal("JIT mode doesn't work with `cargo check`"); + } + + if !tcx.crate_types().contains(&rustc_session::config::CrateType::Executable) { + tcx.dcx().fatal("can't jit non-executable crate"); + } + + let (mut jit_module, mut cx) = create_jit_module( + tcx, + &backend_config, + matches!(backend_config.codegen_mode, CodegenMode::JitLazy), + ); + let mut cached_context = Context::new(); + + let (_, cgus) = tcx.collect_and_partition_mono_items(()); + let mono_items = cgus + .iter() + .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) + .flatten() + .collect::<FxHashMap<_, _>>() + .into_iter() + .collect::<Vec<(_, _)>>(); + + tcx.sess.time("codegen mono items", || { + super::predefine_mono_items(tcx, &mut jit_module, &mono_items); + for (mono_item, _) in mono_items { + match mono_item { + MonoItem::Fn(inst) => match backend_config.codegen_mode { + CodegenMode::Aot => unreachable!(), + CodegenMode::Jit => { + codegen_and_compile_fn( + tcx, + &mut cx, + &mut cached_context, + &mut jit_module, + inst, + ); + } + CodegenMode::JitLazy => { + codegen_shim(tcx, &mut cached_context, &mut jit_module, inst) + } + }, + MonoItem::Static(def_id) => { + crate::constant::codegen_static(tcx, &mut jit_module, def_id); + } + MonoItem::GlobalAsm(item_id) => { + let item = tcx.hir().item(item_id); + tcx.dcx().span_fatal(item.span, "Global asm is not supported in JIT mode"); + } + } + } + }); + + if !cx.global_asm.is_empty() { + tcx.dcx().fatal("Inline asm is not supported in JIT mode"); + } + + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, true, true); + + tcx.dcx().abort_if_errors(); + + jit_module.finalize_definitions(); + + println!( + "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" + ); + + let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) + .chain(backend_config.jit_args.iter().map(|arg| &**arg)) + .map(|arg| CString::new(arg).unwrap()) + .collect::<Vec<_>>(); + + let start_sig = Signature { + params: vec![ + AbiParam::new(jit_module.target_config().pointer_type()), + AbiParam::new(jit_module.target_config().pointer_type()), + ], + returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)], + call_conv: jit_module.target_config().default_call_conv, + }; + let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); + let finalized_start: *const u8 = jit_module.module.get_finalized_function(start_func_id); + + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + assert!(lazy_jit_state.is_none()); + *lazy_jit_state = Some(JitState { jit_module }); + }); + + let f: extern "C" fn(c_int, *const *const c_char) -> c_int = + unsafe { ::std::mem::transmute(finalized_start) }; + + let (tx, rx) = mpsc::channel(); + GLOBAL_MESSAGE_SENDER.set(Mutex::new(tx)).unwrap(); + + // Spawn the jitted runtime in a new thread so that this rustc thread can handle messages + // (eg to lazily JIT further functions as required) + std::thread::spawn(move || { + let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); + + // Push a null pointer as a terminating argument. This is required by POSIX and + // useful as some dynamic linkers use it as a marker to jump over. + argv.push(std::ptr::null()); + + let ret = f(args.len() as c_int, argv.as_ptr()); + std::process::exit(ret); + }); + + // Handle messages + loop { + match rx.recv().unwrap() { + // lazy JIT compilation request - compile requested instance and return pointer to result + UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx } => { + tx.send(jit_fn(instance_ptr, trampoline_ptr)) + .expect("jitted runtime hung up before response to lazy JIT request was sent"); + } + } + } +} + +pub(crate) fn codegen_and_compile_fn<'tcx>( + tcx: TyCtxt<'tcx>, + cx: &mut crate::CodegenCx, + cached_context: &mut Context, + module: &mut dyn Module, + instance: Instance<'tcx>, +) { + cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( + cx.profiler.clone(), + ))); + + tcx.prof.generic_activity("codegen and compile fn").run(|| { + let _inst_guard = + crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); + + let cached_func = std::mem::replace(&mut cached_context.func, Function::new()); + if let Some(codegened_func) = crate::base::codegen_fn( + tcx, + cx, + &mut TypeDebugContext::default(), + cached_func, + module, + instance, + ) { + crate::base::compile_fn(cx, cached_context, module, codegened_func); + } + }); +} + +extern "C" fn clif_jit_fn( + instance_ptr: *const Instance<'static>, + trampoline_ptr: *const u8, +) -> *const u8 { + // send the JIT request to the rustc thread, with a channel for the response + let (tx, rx) = mpsc::channel(); + UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx } + .send() + .expect("rustc thread hung up before lazy JIT request was sent"); + + // block on JIT compilation result + rx.recv().expect("rustc thread hung up before responding to sent lazy JIT request") +} + +fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> *const u8 { + rustc_middle::ty::tls::with(|tcx| { + // lift is used to ensure the correct lifetime for instance. + let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); + + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + let lazy_jit_state = lazy_jit_state.as_mut().unwrap(); + let jit_module = &mut lazy_jit_state.jit_module; + + let name = tcx.symbol_name(instance).name; + let sig = crate::abi::get_function_sig( + tcx, + jit_module.target_config().default_call_conv, + instance, + ); + let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap(); + + let current_ptr = jit_module.module.read_got_entry(func_id); + + // If the function's GOT entry has already been updated to point at something other + // than the shim trampoline, don't re-jit but just return the new pointer instead. + // This does not need synchronization as this code is executed only by a sole rustc + // thread. + if current_ptr != trampoline_ptr { + return current_ptr; + } + + jit_module.module.prepare_for_function_redefine(func_id).unwrap(); + + let mut cx = crate::CodegenCx::new( + tcx, + jit_module.isa(), + false, + Symbol::intern("dummy_cgu_name"), + ); + codegen_and_compile_fn(tcx, &mut cx, &mut Context::new(), jit_module, instance); + + assert!(cx.global_asm.is_empty()); + jit_module.finalize_definitions(); + jit_module.module.get_finalized_function(func_id) + }) + }) +} + +fn dep_symbol_lookup_fn( + sess: &Session, + crate_info: CrateInfo, +) -> Box<dyn Fn(&str) -> Option<*const u8> + Send> { + use rustc_middle::middle::dependency_format::Linkage; + + let mut dylib_paths = Vec::new(); + + let data = &crate_info + .dependency_formats + .iter() + .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable) + .unwrap() + .1; + // `used_crates` is in reverse postorder in terms of dependencies. Reverse the order here to + // get a postorder which ensures that all dependencies of a dylib are loaded before the dylib + // itself. This helps the dynamic linker to find dylibs not in the regular dynamic library + // search path. + for &cnum in crate_info.used_crates.iter().rev() { + let src = &crate_info.used_crate_source[&cnum]; + match data[cnum.as_usize() - 1] { + Linkage::NotLinked | Linkage::IncludedFromDylib => {} + Linkage::Static => { + let name = crate_info.crate_name[&cnum]; + let mut diag = sess.dcx().struct_err(format!("Can't load static lib {}", name)); + diag.note("rustc_codegen_cranelift can only load dylibs in JIT mode."); + diag.emit(); + } + Linkage::Dynamic => { + dylib_paths.push(src.dylib.as_ref().unwrap().0.clone()); + } + } + } + + let imported_dylibs = Box::leak( + dylib_paths + .into_iter() + .map(|path| unsafe { libloading::Library::new(&path).unwrap() }) + .collect::<Box<[_]>>(), + ); + + sess.dcx().abort_if_errors(); + + Box::new(move |sym_name| { + for dylib in &*imported_dylibs { + if let Ok(sym) = unsafe { dylib.get::<*const u8>(sym_name.as_bytes()) } { + return Some(*sym); + } + } + None + }) +} + +fn codegen_shim<'tcx>( + tcx: TyCtxt<'tcx>, + cached_context: &mut Context, + module: &mut UnwindModule<JITModule>, + inst: Instance<'tcx>, +) { + let pointer_type = module.target_config().pointer_type(); + + let name = tcx.symbol_name(inst).name; + let sig = crate::abi::get_function_sig(tcx, module.target_config().default_call_conv, inst); + let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap(); + + let instance_ptr = Box::into_raw(Box::new(inst)); + + let jit_fn = module + .declare_function( + "__clif_jit_fn", + Linkage::Import, + &Signature { + call_conv: module.target_config().default_call_conv, + params: vec![AbiParam::new(pointer_type), AbiParam::new(pointer_type)], + returns: vec![AbiParam::new(pointer_type)], + }, + ) + .unwrap(); + + let context = cached_context; + context.clear(); + let trampoline = &mut context.func; + trampoline.signature = sig.clone(); + + let mut builder_ctx = FunctionBuilderContext::new(); + let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx); + + let trampoline_fn = module.declare_func_in_func(func_id, trampoline_builder.func); + let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func); + let sig_ref = trampoline_builder.func.import_signature(sig); + + let entry_block = trampoline_builder.create_block(); + trampoline_builder.append_block_params_for_function_params(entry_block); + let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec(); + + trampoline_builder.switch_to_block(entry_block); + let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64); + let trampoline_ptr = trampoline_builder.ins().func_addr(pointer_type, trampoline_fn); + let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr, trampoline_ptr]); + let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; + let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args); + let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); + trampoline_builder.ins().return_(&ret_vals); + + module.define_function(func_id, context).unwrap(); +} diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs new file mode 100644 index 00000000000..fb0eed07c19 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs @@ -0,0 +1,84 @@ +//! Drivers are responsible for calling [`codegen_fn`] or [`codegen_static`] for each mono item and +//! performing any further actions like JIT executing or writing object files. +//! +//! [`codegen_fn`]: crate::base::codegen_fn +//! [`codegen_static`]: crate::constant::codegen_static + +use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir::mono::{MonoItem, MonoItemData}; + +use crate::prelude::*; + +pub(crate) mod aot; +#[cfg(feature = "jit")] +pub(crate) mod jit; + +fn predefine_mono_items<'tcx>( + tcx: TyCtxt<'tcx>, + module: &mut dyn Module, + mono_items: &[(MonoItem<'tcx>, MonoItemData)], +) { + tcx.prof.generic_activity("predefine functions").run(|| { + let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE); + for &(mono_item, data) in mono_items { + match mono_item { + MonoItem::Fn(instance) => { + let name = tcx.symbol_name(instance).name; + let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); + let sig = + get_function_sig(tcx, module.target_config().default_call_conv, instance); + let linkage = crate::linkage::get_clif_linkage( + mono_item, + data.linkage, + data.visibility, + is_compiler_builtins, + ); + let is_naked = tcx + .codegen_fn_attrs(instance.def_id()) + .flags + .contains(CodegenFnAttrFlags::NAKED); + module + .declare_function( + name, + // Naked functions are defined in a separate object + // file from the codegen unit rustc expects them to + // be defined in. + if is_naked { Linkage::Import } else { linkage }, + &sig, + ) + .unwrap(); + } + MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} + } + } + }); +} + +struct MeasuremeProfiler(SelfProfilerRef); + +struct TimingGuard { + profiler: std::mem::ManuallyDrop<SelfProfilerRef>, + inner: Option<rustc_data_structures::profiling::TimingGuard<'static>>, +} + +impl Drop for TimingGuard { + fn drop(&mut self) { + self.inner.take(); + unsafe { + std::mem::ManuallyDrop::drop(&mut self.profiler); + } + } +} + +impl cranelift_codegen::timing::Profiler for MeasuremeProfiler { + fn start_pass(&self, pass: cranelift_codegen::timing::Pass) -> Box<dyn std::any::Any> { + let mut timing_guard = + TimingGuard { profiler: std::mem::ManuallyDrop::new(self.0.clone()), inner: None }; + timing_guard.inner = Some( + unsafe { &*(&*timing_guard.profiler as &SelfProfilerRef as *const SelfProfilerRef) } + .generic_activity(pass.description()), + ); + Box::new(timing_guard) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs new file mode 100644 index 00000000000..0c99a5ce12f --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs @@ -0,0 +1,225 @@ +//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a +//! standalone executable. + +use std::io::Write; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::sync::Arc; + +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_hir::{InlineAsmOperand, ItemId}; +use rustc_middle::mir::interpret::ErrorHandled; +use rustc_session::config::{OutputFilenames, OutputType}; +use rustc_target::asm::InlineAsmArch; + +use crate::prelude::*; + +pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) { + let item = tcx.hir().item(item_id); + if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind { + let is_x86 = + matches!(tcx.sess.asm_arch.unwrap(), InlineAsmArch::X86 | InlineAsmArch::X86_64); + + if is_x86 { + if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { + global_asm.push_str("\n.intel_syntax noprefix\n"); + } else { + global_asm.push_str("\n.att_syntax\n"); + } + } + for piece in asm.template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s), + InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: op_sp } => { + match asm.operands[operand_idx].0 { + InlineAsmOperand::Const { ref anon_const } => { + match tcx.const_eval_poly(anon_const.def_id.to_def_id()) { + Ok(const_value) => { + let ty = tcx + .typeck_body(anon_const.body) + .node_type(anon_const.hir_id); + let string = rustc_codegen_ssa::common::asm_const_to_str( + tcx, + op_sp, + const_value, + RevealAllLayoutCx(tcx).layout_of(ty), + ); + global_asm.push_str(&string); + } + Err(ErrorHandled::Reported { .. }) => { + // An error has already been reported and compilation is + // guaranteed to fail if execution hits this path. + } + Err(ErrorHandled::TooGeneric(_)) => { + span_bug!(op_sp, "asm const cannot be resolved; too generic"); + } + } + } + InlineAsmOperand::SymFn { anon_const } => { + if cfg!(not(feature = "inline_asm_sym")) { + tcx.dcx().span_err( + item.span, + "asm! and global_asm! sym operands are not yet supported", + ); + } + + let ty = tcx.typeck_body(anon_const.body).node_type(anon_const.hir_id); + let instance = match ty.kind() { + &ty::FnDef(def_id, args) => Instance::new(def_id, args), + _ => span_bug!(op_sp, "asm sym is not a function"), + }; + let symbol = tcx.symbol_name(instance); + // FIXME handle the case where the function was made private to the + // current codegen unit + global_asm.push_str(symbol.name); + } + InlineAsmOperand::SymStatic { path: _, def_id } => { + if cfg!(not(feature = "inline_asm_sym")) { + tcx.dcx().span_err( + item.span, + "asm! and global_asm! sym operands are not yet supported", + ); + } + + let instance = Instance::mono(tcx, def_id); + let symbol = tcx.symbol_name(instance); + global_asm.push_str(symbol.name); + } + InlineAsmOperand::In { .. } + | InlineAsmOperand::Out { .. } + | InlineAsmOperand::InOut { .. } + | InlineAsmOperand::SplitInOut { .. } + | InlineAsmOperand::Label { .. } => { + span_bug!(op_sp, "invalid operand type for global_asm!") + } + } + } + } + } + + global_asm.push('\n'); + if is_x86 { + global_asm.push_str(".att_syntax\n\n"); + } + } else { + bug!("Expected GlobalAsm found {:?}", item); + } +} + +#[derive(Debug)] +pub(crate) struct GlobalAsmConfig { + assembler: PathBuf, + target: String, + pub(crate) output_filenames: Arc<OutputFilenames>, +} + +impl GlobalAsmConfig { + pub(crate) fn new(tcx: TyCtxt<'_>) -> Self { + GlobalAsmConfig { + assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"), + target: match &tcx.sess.opts.target_triple { + rustc_target::spec::TargetTriple::TargetTriple(triple) => triple.clone(), + rustc_target::spec::TargetTriple::TargetJson { path_for_rustdoc, .. } => { + path_for_rustdoc.to_str().unwrap().to_owned() + } + }, + output_filenames: tcx.output_filenames(()).clone(), + } + } +} + +pub(crate) fn compile_global_asm( + config: &GlobalAsmConfig, + cgu_name: &str, + global_asm: &str, +) -> Result<Option<PathBuf>, String> { + if global_asm.is_empty() { + return Ok(None); + } + + // Remove all LLVM style comments + let mut global_asm = global_asm + .lines() + .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line }) + .collect::<Vec<_>>() + .join("\n"); + global_asm.push('\n'); + + let global_asm_object_file = add_file_stem_postfix( + config.output_filenames.temp_path(OutputType::Object, Some(cgu_name)), + ".asm", + ); + + // Assemble `global_asm` + if option_env!("CG_CLIF_FORCE_GNU_AS").is_some() { + let mut child = Command::new(&config.assembler) + .arg("-o") + .arg(&global_asm_object_file) + .stdin(Stdio::piped()) + .spawn() + .expect("Failed to spawn `as`."); + child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); + let status = child.wait().expect("Failed to wait for `as`."); + if !status.success() { + return Err(format!("Failed to assemble `{}`", global_asm)); + } + } else { + let mut child = Command::new(std::env::current_exe().unwrap()) + // Avoid a warning about the jobserver fd not being passed + .env_remove("CARGO_MAKEFLAGS") + .arg("--target") + .arg(&config.target) + .arg("--crate-type") + .arg("staticlib") + .arg("--emit") + .arg("obj") + .arg("-o") + .arg(&global_asm_object_file) + .arg("-") + .arg("-Abad_asm_style") + .arg("-Zcodegen-backend=llvm") + .stdin(Stdio::piped()) + .spawn() + .expect("Failed to spawn `as`."); + let mut stdin = child.stdin.take().unwrap(); + stdin + .write_all( + br####" + #![feature(decl_macro, no_core, rustc_attrs)] + #![allow(internal_features)] + #![no_core] + #[rustc_builtin_macro] + #[rustc_macro_transparency = "semitransparent"] + macro global_asm() { /* compiler built-in */ } + global_asm!(r###" + "####, + ) + .unwrap(); + stdin.write_all(global_asm.as_bytes()).unwrap(); + stdin + .write_all( + br####" + "###); + "####, + ) + .unwrap(); + std::mem::drop(stdin); + let status = child.wait().expect("Failed to wait for `as`."); + if !status.success() { + return Err(format!("Failed to assemble `{}`", global_asm)); + } + } + + Ok(Some(global_asm_object_file)) +} + +pub(crate) fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { + let mut new_filename = path.file_stem().unwrap().to_owned(); + new_filename.push(postfix); + if let Some(extension) = path.extension() { + new_filename.push("."); + new_filename.push(extension); + } + path.set_file_name(new_filename); + path +} diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs new file mode 100644 index 00000000000..16edec47e10 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -0,0 +1,933 @@ +//! Codegen of `asm!` invocations. + +use std::fmt::Write; + +use cranelift_codegen::isa::CallConv; +use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_hir::LangItem; +use rustc_span::sym; +use rustc_target::asm::*; +use target_lexicon::BinaryFormat; + +use crate::prelude::*; + +pub(crate) enum CInlineAsmOperand<'tcx> { + In { + reg: InlineAsmRegOrRegClass, + value: Value, + }, + Out { + reg: InlineAsmRegOrRegClass, + late: bool, + place: Option<CPlace<'tcx>>, + }, + InOut { + reg: InlineAsmRegOrRegClass, + _late: bool, + in_value: Value, + out_place: Option<CPlace<'tcx>>, + }, + Const { + value: String, + }, + Symbol { + symbol: String, + }, +} + +pub(crate) fn codegen_inline_asm_terminator<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + span: Span, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperand<'tcx>], + options: InlineAsmOptions, + destination: Option<mir::BasicBlock>, +) { + // Used by panic_abort on Windows, but uses a syntax which only happens to work with + // asm!() by accident and breaks with the GNU assembler as well as global_asm!() for + // the LLVM backend. + if template.len() == 1 && template[0] == InlineAsmTemplatePiece::String("int $$0x29".into()) { + fx.bcx.ins().trap(TrapCode::User(1)); + return; + } + + let operands = operands + .iter() + .map(|operand| match *operand { + InlineAsmOperand::In { reg, ref value } => CInlineAsmOperand::In { + reg, + value: crate::base::codegen_operand(fx, value).load_scalar(fx), + }, + InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out { + reg, + late, + place: place.map(|place| crate::base::codegen_place(fx, place)), + }, + InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => { + CInlineAsmOperand::InOut { + reg, + _late: late, + in_value: crate::base::codegen_operand(fx, in_value).load_scalar(fx), + out_place: out_place.map(|place| crate::base::codegen_place(fx, place)), + } + } + InlineAsmOperand::Const { ref value } => { + let (const_value, ty) = crate::constant::eval_mir_constant(fx, value); + let value = rustc_codegen_ssa::common::asm_const_to_str( + fx.tcx, + span, + const_value, + fx.layout_of(ty), + ); + CInlineAsmOperand::Const { value } + } + InlineAsmOperand::SymFn { ref value } => { + if cfg!(not(feature = "inline_asm_sym")) { + fx.tcx + .dcx() + .span_err(span, "asm! and global_asm! sym operands are not yet supported"); + } + + let const_ = fx.monomorphize(value.const_); + if let ty::FnDef(def_id, args) = *const_.ty().kind() { + let instance = ty::Instance::resolve_for_fn_ptr( + fx.tcx, + ty::ParamEnv::reveal_all(), + def_id, + args, + ) + .unwrap(); + let symbol = fx.tcx.symbol_name(instance); + + // Pass a wrapper rather than the function itself as the function itself may not + // be exported from the main codegen unit and may thus be unreachable from the + // object file created by an external assembler. + let inline_asm_index = fx.cx.inline_asm_index.get(); + fx.cx.inline_asm_index.set(inline_asm_index + 1); + let wrapper_name = format!( + "__inline_asm_{}_wrapper_n{}", + fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), + inline_asm_index + ); + let sig = + get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance); + create_wrapper_function(fx.module, sig, &wrapper_name, symbol.name); + + CInlineAsmOperand::Symbol { symbol: wrapper_name } + } else { + span_bug!(span, "invalid type for asm sym (fn)"); + } + } + InlineAsmOperand::SymStatic { def_id } => { + assert!(fx.tcx.is_static(def_id)); + let instance = Instance::mono(fx.tcx, def_id); + CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() } + } + InlineAsmOperand::Label { .. } => { + span_bug!(span, "asm! label operands are not yet supported"); + } + }) + .collect::<Vec<_>>(); + + codegen_inline_asm_inner(fx, template, &operands, options); + + match destination { + Some(destination) => { + let destination_block = fx.get_block(destination); + fx.bcx.ins().jump(destination_block, &[]); + } + None => { + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + } + } +} + +pub(crate) fn codegen_inline_asm_inner<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + template: &[InlineAsmTemplatePiece], + operands: &[CInlineAsmOperand<'tcx>], + options: InlineAsmOptions, +) { + // FIXME add .eh_frame unwind info directives + + let mut asm_gen = InlineAssemblyGenerator { + tcx: fx.tcx, + arch: fx.tcx.sess.asm_arch.unwrap(), + enclosing_def_id: fx.instance.def_id(), + template, + operands, + options, + registers: Vec::new(), + stack_slots_clobber: Vec::new(), + stack_slots_input: Vec::new(), + stack_slots_output: Vec::new(), + stack_slot_size: Size::from_bytes(0), + is_naked: false, + }; + asm_gen.allocate_registers(); + asm_gen.allocate_stack_slots(); + + let inline_asm_index = fx.cx.inline_asm_index.get(); + fx.cx.inline_asm_index.set(inline_asm_index + 1); + let asm_name = format!( + "__inline_asm_{}_n{}", + fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), + inline_asm_index + ); + + let generated_asm = asm_gen.generate_asm_wrapper(&asm_name); + fx.cx.global_asm.push_str(&generated_asm); + + let mut inputs = Vec::new(); + let mut outputs = Vec::new(); + for (i, operand) in operands.iter().enumerate() { + match operand { + CInlineAsmOperand::In { reg: _, value } => { + inputs.push((asm_gen.stack_slots_input[i].unwrap(), *value)); + } + CInlineAsmOperand::Out { reg: _, late: _, place } => { + if let Some(place) = place { + outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place)); + } + } + CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => { + inputs.push((asm_gen.stack_slots_input[i].unwrap(), *in_value)); + if let Some(out_place) = out_place { + outputs.push((asm_gen.stack_slots_output[i].unwrap(), *out_place)); + } + } + CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {} + } + } + + call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs); +} + +pub(crate) fn codegen_naked_asm<'tcx>( + tcx: TyCtxt<'tcx>, + cx: &mut crate::CodegenCx, + module: &mut dyn Module, + instance: Instance<'tcx>, + span: Span, + symbol_name: &str, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperand<'tcx>], + options: InlineAsmOptions, +) { + // FIXME add .eh_frame unwind info directives + + let operands = operands + .iter() + .map(|operand| match *operand { + InlineAsmOperand::In { .. } + | InlineAsmOperand::Out { .. } + | InlineAsmOperand::InOut { .. } => { + span_bug!(span, "invalid operand type for naked asm") + } + InlineAsmOperand::Const { ref value } => { + let cv = instance.instantiate_mir_and_normalize_erasing_regions( + tcx, + ty::ParamEnv::reveal_all(), + ty::EarlyBinder::bind(value.const_), + ); + let const_value = cv + .eval(tcx, ty::ParamEnv::reveal_all(), value.span) + .expect("erroneous constant missed by mono item collection"); + + let value = rustc_codegen_ssa::common::asm_const_to_str( + tcx, + span, + const_value, + RevealAllLayoutCx(tcx).layout_of(cv.ty()), + ); + CInlineAsmOperand::Const { value } + } + InlineAsmOperand::SymFn { ref value } => { + if cfg!(not(feature = "inline_asm_sym")) { + tcx.dcx() + .span_err(span, "asm! and global_asm! sym operands are not yet supported"); + } + + let const_ = instance.instantiate_mir_and_normalize_erasing_regions( + tcx, + ty::ParamEnv::reveal_all(), + ty::EarlyBinder::bind(value.const_), + ); + if let ty::FnDef(def_id, args) = *const_.ty().kind() { + let instance = ty::Instance::resolve_for_fn_ptr( + tcx, + ty::ParamEnv::reveal_all(), + def_id, + args, + ) + .unwrap(); + let symbol = tcx.symbol_name(instance); + + // Pass a wrapper rather than the function itself as the function itself may not + // be exported from the main codegen unit and may thus be unreachable from the + // object file created by an external assembler. + let inline_asm_index = cx.inline_asm_index.get(); + cx.inline_asm_index.set(inline_asm_index + 1); + let wrapper_name = format!( + "__inline_asm_{}_wrapper_n{}", + cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), + inline_asm_index + ); + let sig = + get_function_sig(tcx, module.target_config().default_call_conv, instance); + create_wrapper_function(module, sig, &wrapper_name, symbol.name); + + CInlineAsmOperand::Symbol { symbol: wrapper_name } + } else { + span_bug!(span, "invalid type for asm sym (fn)"); + } + } + InlineAsmOperand::SymStatic { def_id } => { + assert!(tcx.is_static(def_id)); + let instance = Instance::mono(tcx, def_id); + CInlineAsmOperand::Symbol { symbol: tcx.symbol_name(instance).name.to_owned() } + } + InlineAsmOperand::Label { .. } => { + span_bug!(span, "asm! label operands are not yet supported"); + } + }) + .collect::<Vec<_>>(); + + let asm_gen = InlineAssemblyGenerator { + tcx, + arch: tcx.sess.asm_arch.unwrap(), + enclosing_def_id: instance.def_id(), + template, + operands: &operands, + options, + registers: Vec::new(), + stack_slots_clobber: Vec::new(), + stack_slots_input: Vec::new(), + stack_slots_output: Vec::new(), + stack_slot_size: Size::from_bytes(0), + is_naked: true, + }; + + let generated_asm = asm_gen.generate_asm_wrapper(symbol_name); + cx.global_asm.push_str(&generated_asm); +} + +struct InlineAssemblyGenerator<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + arch: InlineAsmArch, + enclosing_def_id: DefId, + template: &'a [InlineAsmTemplatePiece], + operands: &'a [CInlineAsmOperand<'tcx>], + options: InlineAsmOptions, + registers: Vec<Option<InlineAsmReg>>, + stack_slots_clobber: Vec<Option<Size>>, + stack_slots_input: Vec<Option<Size>>, + stack_slots_output: Vec<Option<Size>>, + stack_slot_size: Size, + is_naked: bool, +} + +impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { + fn allocate_registers(&mut self) { + assert!(!self.is_naked); + + let sess = self.tcx.sess; + let map = allocatable_registers( + self.arch, + sess.relocation_model(), + self.tcx.asm_target_features(self.enclosing_def_id), + &sess.target, + ); + let mut allocated = FxHashMap::<_, (bool, bool)>::default(); + let mut regs = vec![None; self.operands.len()]; + + // Add explicit registers to the allocated set. + for (i, operand) in self.operands.iter().enumerate() { + match *operand { + CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => { + regs[i] = Some(reg); + allocated.entry(reg).or_default().0 = true; + } + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(reg), + late: true, + .. + } => { + regs[i] = Some(reg); + allocated.entry(reg).or_default().1 = true; + } + CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. } + | CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => { + regs[i] = Some(reg); + allocated.insert(reg, (true, true)); + } + _ => (), + } + } + + // Allocate out/inout/inlateout registers first because they are more constrained. + for (i, operand) in self.operands.iter().enumerate() { + match *operand { + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::RegClass(class), + late: false, + .. + } + | CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::RegClass(class), .. + } => { + let mut alloc_reg = None; + for ® in &map[&class] { + let mut used = false; + reg.overlapping_regs(|r| { + if allocated.contains_key(&r) { + used = true; + } + }); + + if !used { + alloc_reg = Some(reg); + break; + } + } + + let reg = alloc_reg.expect("cannot allocate registers"); + regs[i] = Some(reg); + allocated.insert(reg, (true, true)); + } + _ => (), + } + } + + // Allocate in/lateout. + for (i, operand) in self.operands.iter().enumerate() { + match *operand { + CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => { + let mut alloc_reg = None; + for ® in &map[&class] { + let mut used = false; + reg.overlapping_regs(|r| { + if allocated.get(&r).copied().unwrap_or_default().0 { + used = true; + } + }); + + if !used { + alloc_reg = Some(reg); + break; + } + } + + let reg = alloc_reg.expect("cannot allocate registers"); + regs[i] = Some(reg); + allocated.entry(reg).or_default().0 = true; + } + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::RegClass(class), + late: true, + .. + } => { + let mut alloc_reg = None; + for ® in &map[&class] { + let mut used = false; + reg.overlapping_regs(|r| { + if allocated.get(&r).copied().unwrap_or_default().1 { + used = true; + } + }); + + if !used { + alloc_reg = Some(reg); + break; + } + } + + let reg = alloc_reg.expect("cannot allocate registers"); + regs[i] = Some(reg); + allocated.entry(reg).or_default().1 = true; + } + _ => (), + } + } + + self.registers = regs; + } + + fn allocate_stack_slots(&mut self) { + assert!(!self.is_naked); + + let mut slot_size = Size::from_bytes(0); + let mut slots_clobber = vec![None; self.operands.len()]; + let mut slots_input = vec![None; self.operands.len()]; + let mut slots_output = vec![None; self.operands.len()]; + + let new_slot_fn = |slot_size: &mut Size, reg_class: InlineAsmRegClass| { + let reg_size = + reg_class.supported_types(self.arch).iter().map(|(ty, _)| ty.size()).max().unwrap(); + let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap(); + let offset = slot_size.align_to(align); + *slot_size = offset + reg_size; + offset + }; + let mut new_slot = |x| new_slot_fn(&mut slot_size, x); + + // Allocate stack slots for saving clobbered registers + let abi_clobber = InlineAsmClobberAbi::parse(self.arch, &self.tcx.sess.target, sym::C) + .unwrap() + .clobbered_regs(); + for (i, reg) in self.registers.iter().enumerate().filter_map(|(i, r)| r.map(|r| (i, r))) { + let mut need_save = true; + // If the register overlaps with a register clobbered by function call, then + // we don't need to save it. + for r in abi_clobber { + r.overlapping_regs(|r| { + if r == reg { + need_save = false; + } + }); + + if !need_save { + break; + } + } + + if need_save { + slots_clobber[i] = Some(new_slot(reg.reg_class())); + } + } + + // Allocate stack slots for inout + for (i, operand) in self.operands.iter().enumerate() { + match *operand { + CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => { + let slot = new_slot(reg.reg_class()); + slots_input[i] = Some(slot); + slots_output[i] = Some(slot); + } + _ => (), + } + } + + let slot_size_before_input = slot_size; + let mut new_slot = |x| new_slot_fn(&mut slot_size, x); + + // Allocate stack slots for input + for (i, operand) in self.operands.iter().enumerate() { + match *operand { + CInlineAsmOperand::In { reg, .. } + | CInlineAsmOperand::InOut { reg, out_place: None, .. } => { + slots_input[i] = Some(new_slot(reg.reg_class())); + } + _ => (), + } + } + + // Reset slot size to before input so that input and output operands can overlap + // and save some memory. + let slot_size_after_input = slot_size; + slot_size = slot_size_before_input; + let mut new_slot = |x| new_slot_fn(&mut slot_size, x); + + // Allocate stack slots for output + for (i, operand) in self.operands.iter().enumerate() { + match *operand { + CInlineAsmOperand::Out { reg, place: Some(_), .. } => { + slots_output[i] = Some(new_slot(reg.reg_class())); + } + _ => (), + } + } + + slot_size = slot_size.max(slot_size_after_input); + + self.stack_slots_clobber = slots_clobber; + self.stack_slots_input = slots_input; + self.stack_slots_output = slots_output; + self.stack_slot_size = slot_size; + } + + fn generate_asm_wrapper(&self, asm_name: &str) -> String { + let binary_format = crate::target_triple(self.tcx.sess).binary_format; + + let mut generated_asm = String::new(); + match binary_format { + BinaryFormat::Elf => { + writeln!(generated_asm, ".globl {}", asm_name).unwrap(); + writeln!(generated_asm, ".type {},@function", asm_name).unwrap(); + writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap(); + writeln!(generated_asm, "{}:", asm_name).unwrap(); + } + BinaryFormat::Macho => { + writeln!(generated_asm, ".globl _{}", asm_name).unwrap(); + writeln!(generated_asm, "_{}:", asm_name).unwrap(); + } + BinaryFormat::Coff => { + writeln!(generated_asm, ".globl {}", asm_name).unwrap(); + writeln!(generated_asm, "{}:", asm_name).unwrap(); + } + _ => self + .tcx + .dcx() + .fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")), + } + + let is_x86 = matches!(self.arch, InlineAsmArch::X86 | InlineAsmArch::X86_64); + + if is_x86 { + generated_asm.push_str(".intel_syntax noprefix\n"); + } + if !self.is_naked { + Self::prologue(&mut generated_asm, self.arch); + + // Save clobbered registers + if !self.options.contains(InlineAsmOptions::NORETURN) { + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_clobber.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::save_register(&mut generated_asm, self.arch, reg, slot); + } + } + + // Write input registers + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_input.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::restore_register(&mut generated_asm, self.arch, reg, slot); + } + } + + if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) { + generated_asm.push_str(".att_syntax\n"); + } + + // The actual inline asm + for piece in self.template { + match piece { + InlineAsmTemplatePiece::String(s) => { + generated_asm.push_str(s); + } + InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { + match self.operands[*operand_idx] { + CInlineAsmOperand::In { .. } + | CInlineAsmOperand::Out { .. } + | CInlineAsmOperand::InOut { .. } => { + if self.options.contains(InlineAsmOptions::ATT_SYNTAX) { + generated_asm.push('%'); + } + + let reg = self.registers[*operand_idx].unwrap(); + match self.arch { + InlineAsmArch::X86_64 => match reg { + InlineAsmReg::X86(reg) + if reg as u32 >= X86InlineAsmReg::xmm0 as u32 + && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => + { + // rustc emits x0 rather than xmm0 + let class = match *modifier { + None | Some('x') => "xmm", + Some('y') => "ymm", + Some('z') => "zmm", + _ => unreachable!(), + }; + write!( + generated_asm, + "{class}{}", + reg as u32 - X86InlineAsmReg::xmm0 as u32 + ) + .unwrap(); + } + _ => reg + .emit(&mut generated_asm, InlineAsmArch::X86_64, *modifier) + .unwrap(), + }, + _ => reg.emit(&mut generated_asm, self.arch, *modifier).unwrap(), + } + } + CInlineAsmOperand::Const { ref value } => { + generated_asm.push_str(value); + } + CInlineAsmOperand::Symbol { ref symbol } => generated_asm.push_str(symbol), + } + } + } + } + generated_asm.push('\n'); + + if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) { + generated_asm.push_str(".intel_syntax noprefix\n"); + } + + if !self.is_naked { + if !self.options.contains(InlineAsmOptions::NORETURN) { + // Read output registers + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_output.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::save_register(&mut generated_asm, self.arch, reg, slot); + } + + // Restore clobbered registers + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_clobber.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::restore_register(&mut generated_asm, self.arch, reg, slot); + } + + Self::epilogue(&mut generated_asm, self.arch); + } else { + Self::epilogue_noreturn(&mut generated_asm, self.arch); + } + } + + if is_x86 { + generated_asm.push_str(".att_syntax\n"); + } + + match binary_format { + BinaryFormat::Elf => { + writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap(); + generated_asm.push_str(".text\n"); + } + BinaryFormat::Macho | BinaryFormat::Coff => {} + _ => self + .tcx + .dcx() + .fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")), + } + + generated_asm.push_str("\n\n"); + + generated_asm + } + + fn prologue(generated_asm: &mut String, arch: InlineAsmArch) { + match arch { + InlineAsmArch::X86_64 => { + generated_asm.push_str(" push rbp\n"); + generated_asm.push_str(" mov rbp,rsp\n"); + generated_asm.push_str(" push rbx\n"); // rbx is callee saved + // rbx is reserved by LLVM for the "base pointer", so rustc doesn't allow using it + generated_asm.push_str(" mov rbx,rdi\n"); + } + InlineAsmArch::AArch64 => { + generated_asm.push_str(" stp fp, lr, [sp, #-32]!\n"); + generated_asm.push_str(" mov fp, sp\n"); + generated_asm.push_str(" str x19, [sp, #24]\n"); // x19 is callee saved + // x19 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it + generated_asm.push_str(" mov x19, x0\n"); + } + InlineAsmArch::RiscV64 => { + generated_asm.push_str(" addi sp, sp, -16\n"); + generated_asm.push_str(" sd ra, 8(sp)\n"); + generated_asm.push_str(" sd s1, 0(sp)\n"); // s1 is callee saved + // s1/x9 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it + generated_asm.push_str(" mv s1, a0\n"); + } + _ => unimplemented!("prologue for {:?}", arch), + } + } + + fn epilogue(generated_asm: &mut String, arch: InlineAsmArch) { + match arch { + InlineAsmArch::X86_64 => { + generated_asm.push_str(" pop rbx\n"); + generated_asm.push_str(" pop rbp\n"); + generated_asm.push_str(" ret\n"); + } + InlineAsmArch::AArch64 => { + generated_asm.push_str(" ldr x19, [sp, #24]\n"); + generated_asm.push_str(" ldp fp, lr, [sp], #32\n"); + generated_asm.push_str(" ret\n"); + } + InlineAsmArch::RiscV64 => { + generated_asm.push_str(" ld s1, 0(sp)\n"); + generated_asm.push_str(" ld ra, 8(sp)\n"); + generated_asm.push_str(" addi sp, sp, 16\n"); + generated_asm.push_str(" ret\n"); + } + _ => unimplemented!("epilogue for {:?}", arch), + } + } + + fn epilogue_noreturn(generated_asm: &mut String, arch: InlineAsmArch) { + match arch { + InlineAsmArch::X86_64 => { + generated_asm.push_str(" ud2\n"); + } + InlineAsmArch::AArch64 => { + generated_asm.push_str(" brk #0x1\n"); + } + InlineAsmArch::RiscV64 => { + generated_asm.push_str(" ebreak\n"); + } + _ => unimplemented!("epilogue_noreturn for {:?}", arch), + } + } + + fn save_register( + generated_asm: &mut String, + arch: InlineAsmArch, + reg: InlineAsmReg, + offset: Size, + ) { + match arch { + InlineAsmArch::X86_64 => { + match reg { + InlineAsmReg::X86(reg) + if reg as u32 >= X86InlineAsmReg::xmm0 as u32 + && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => + { + // rustc emits x0 rather than xmm0 + write!(generated_asm, " movups [rbx+0x{:x}], ", offset.bytes()).unwrap(); + write!(generated_asm, "xmm{}", reg as u32 - X86InlineAsmReg::xmm0 as u32) + .unwrap(); + } + _ => { + write!(generated_asm, " mov [rbx+0x{:x}], ", offset.bytes()).unwrap(); + reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap(); + } + } + generated_asm.push('\n'); + } + InlineAsmArch::AArch64 => { + generated_asm.push_str(" str "); + reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap(); + writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap(); + } + InlineAsmArch::RiscV64 => { + generated_asm.push_str(" sd "); + reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap(); + writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap(); + } + _ => unimplemented!("save_register for {:?}", arch), + } + } + + fn restore_register( + generated_asm: &mut String, + arch: InlineAsmArch, + reg: InlineAsmReg, + offset: Size, + ) { + match arch { + InlineAsmArch::X86_64 => { + match reg { + InlineAsmReg::X86(reg) + if reg as u32 >= X86InlineAsmReg::xmm0 as u32 + && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => + { + // rustc emits x0 rather than xmm0 + write!( + generated_asm, + " movups xmm{}", + reg as u32 - X86InlineAsmReg::xmm0 as u32 + ) + .unwrap(); + } + _ => { + generated_asm.push_str(" mov "); + reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap() + } + } + writeln!(generated_asm, ", [rbx+0x{:x}]", offset.bytes()).unwrap(); + } + InlineAsmArch::AArch64 => { + generated_asm.push_str(" ldr "); + reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap(); + writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap(); + } + InlineAsmArch::RiscV64 => { + generated_asm.push_str(" ld "); + reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap(); + writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap(); + } + _ => unimplemented!("restore_register for {:?}", arch), + } + } +} + +fn call_inline_asm<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + asm_name: &str, + slot_size: Size, + inputs: Vec<(Size, Value)>, + outputs: Vec<(Size, CPlace<'tcx>)>, +) { + let stack_slot = fx.create_stack_slot(u32::try_from(slot_size.bytes()).unwrap(), 16); + + let inline_asm_func = fx + .module + .declare_function( + asm_name, + Linkage::Import, + &Signature { + call_conv: CallConv::SystemV, + params: vec![AbiParam::new(fx.pointer_type)], + returns: vec![], + }, + ) + .unwrap(); + let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(inline_asm_func, asm_name); + } + + for (offset, value) in inputs { + stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).store( + fx, + value, + MemFlags::trusted(), + ); + } + + let stack_slot_addr = stack_slot.get_addr(fx); + fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]); + + for (offset, place) in outputs { + let ty = if place.layout().ty.is_simd() { + let (lane_count, lane_type) = place.layout().ty.simd_size_and_type(fx.tcx); + asm_clif_type(fx, lane_type).unwrap().by(lane_count.try_into().unwrap()).unwrap() + } else { + asm_clif_type(fx, place.layout().ty).unwrap() + }; + let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load( + fx, + ty, + MemFlags::trusted(), + ); + place.write_cvalue(fx, CValue::by_val(value, place.layout())); + } +} + +fn asm_clif_type<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> Option<types::Type> { + match ty.kind() { + // Adapted from https://github.com/rust-lang/rust/blob/f3c66088610c1b80110297c2d9a8b5f9265b013f/compiler/rustc_hir_analysis/src/check/intrinsicck.rs#L136-L151 + ty::Adt(adt, args) if fx.tcx.is_lang_item(adt.did(), LangItem::MaybeUninit) => { + let fields = &adt.non_enum_variant().fields; + let ty = fields[FieldIdx::from_u32(1)].ty(fx.tcx, args); + let ty::Adt(ty, args) = ty.kind() else { + unreachable!("expected first field of `MaybeUninit` to be an ADT") + }; + assert!( + ty.is_manually_drop(), + "expected first field of `MaybeUninit` to be `ManuallyDrop`" + ); + let fields = &ty.non_enum_variant().fields; + let ty = fields[FieldIdx::ZERO].ty(fx.tcx, args); + fx.clif_type(ty) + } + _ => fx.clif_type(ty), + } +} diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs new file mode 100644 index 00000000000..720a0d8fbf5 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs @@ -0,0 +1,69 @@ +//! Emulate LLVM intrinsics + +use crate::intrinsics::*; +use crate::prelude::*; + +pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + intrinsic: &str, + args: &[Spanned<mir::Operand<'tcx>>], + ret: CPlace<'tcx>, + target: Option<BasicBlock>, + span: Span, +) { + if intrinsic.starts_with("llvm.aarch64") { + return llvm_aarch64::codegen_aarch64_llvm_intrinsic_call(fx, intrinsic, args, ret, target); + } + if intrinsic.starts_with("llvm.x86") { + return llvm_x86::codegen_x86_llvm_intrinsic_call(fx, intrinsic, args, ret, target, span); + } + + match intrinsic { + "llvm.prefetch" => { + // Nothing to do. This is merely a perf hint. + } + + _ if intrinsic.starts_with("llvm.ctlz.v") => { + intrinsic_args!(fx, args => (a); intrinsic); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().clz(lane) + }); + } + + _ if intrinsic.starts_with("llvm.ctpop.v") => { + intrinsic_args!(fx, args => (a); intrinsic); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().popcnt(lane) + }); + } + + _ if intrinsic.starts_with("llvm.fma.v") => { + intrinsic_args!(fx, args => (x,y,z); intrinsic); + + simd_trio_for_each_lane( + fx, + x, + y, + z, + ret, + &|fx, _lane_ty, _res_lane_ty, lane_x, lane_y, lane_z| { + fx.bcx.ins().fma(lane_x, lane_y, lane_z) + }, + ); + } + + _ => { + fx.tcx + .dcx() + .warn(format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic)); + crate::trap::trap_unimplemented(fx, intrinsic); + return; + } + } + + let dest = target.expect("all llvm intrinsics used by stdlib should return"); + let ret_block = fx.get_block(dest); + fx.bcx.ins().jump(ret_block, &[]); +} diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs new file mode 100644 index 00000000000..f0fb18608e0 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs @@ -0,0 +1,337 @@ +//! Emulate AArch64 LLVM intrinsics + +use crate::intrinsics::*; +use crate::prelude::*; + +pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + intrinsic: &str, + args: &[Spanned<mir::Operand<'tcx>>], + ret: CPlace<'tcx>, + target: Option<BasicBlock>, +) { + // llvm.aarch64.neon.sqshl.v*i* + + match intrinsic { + "llvm.aarch64.isb" => { + fx.bcx.ins().fence(); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.abs.v") => { + intrinsic_args!(fx, args => (a); intrinsic); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().iabs(lane) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.cls.v") => { + intrinsic_args!(fx, args => (a); intrinsic); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().cls(lane) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.rbit.v") => { + intrinsic_args!(fx, args => (a); intrinsic); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().bitrev(lane) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.sqadd.v") + || intrinsic.starts_with("llvm.aarch64.neon.uqadd.v") => + { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_pair_for_each_lane_typed(fx, x, y, ret, &|fx, x_lane, y_lane| { + crate::num::codegen_saturating_int_binop(fx, BinOp::Add, x_lane, y_lane) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.sqsub.v") + || intrinsic.starts_with("llvm.aarch64.neon.uqsub.v") => + { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_pair_for_each_lane_typed(fx, x, y, ret, &|fx, x_lane, y_lane| { + crate::num::codegen_saturating_int_binop(fx, BinOp::Sub, x_lane, y_lane) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.smax.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| { + let gt = fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane); + fx.bcx.ins().select(gt, x_lane, y_lane) + }, + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.umax.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| { + let gt = fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane); + fx.bcx.ins().select(gt, x_lane, y_lane) + }, + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.smaxv.i") => { + intrinsic_args!(fx, args => (v); intrinsic); + + simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| { + let gt = fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b); + fx.bcx.ins().select(gt, a, b) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.umaxv.i") => { + intrinsic_args!(fx, args => (v); intrinsic); + + simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| { + let gt = fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b); + fx.bcx.ins().select(gt, a, b) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.smin.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| { + let gt = fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane); + fx.bcx.ins().select(gt, x_lane, y_lane) + }, + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.umin.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| { + let gt = fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane); + fx.bcx.ins().select(gt, x_lane, y_lane) + }, + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.sminv.i") => { + intrinsic_args!(fx, args => (v); intrinsic); + + simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| { + let gt = fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b); + fx.bcx.ins().select(gt, a, b) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.uminv.i") => { + intrinsic_args!(fx, args => (v); intrinsic); + + simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| { + let gt = fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b); + fx.bcx.ins().select(gt, a, b) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.umaxp.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_horizontal_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| fx.bcx.ins().umax(x_lane, y_lane), + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.smaxp.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_horizontal_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| fx.bcx.ins().smax(x_lane, y_lane), + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.uminp.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_horizontal_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| fx.bcx.ins().umin(x_lane, y_lane), + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.sminp.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_horizontal_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| fx.bcx.ins().smin(x_lane, y_lane), + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.fminp.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_horizontal_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| fx.bcx.ins().fmin(x_lane, y_lane), + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.fmaxp.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_horizontal_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| fx.bcx.ins().fmax(x_lane, y_lane), + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.addp.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_horizontal_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| fx.bcx.ins().iadd(x_lane, y_lane), + ); + } + + // FIXME generalize vector types + "llvm.aarch64.neon.tbl1.v8i8" => { + intrinsic_args!(fx, args => (t, idx); intrinsic); + + let zero = fx.bcx.ins().iconst(types::I8, 0); + for i in 0..8 { + let idx_lane = idx.value_lane(fx, i).load_scalar(fx); + let is_zero = + fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, idx_lane, 16); + let t_idx = fx.bcx.ins().uextend(fx.pointer_type, idx_lane); + let t_lane = t.value_lane_dyn(fx, t_idx).load_scalar(fx); + let res = fx.bcx.ins().select(is_zero, zero, t_lane); + ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); + } + } + "llvm.aarch64.neon.tbl1.v16i8" => { + intrinsic_args!(fx, args => (t, idx); intrinsic); + + let zero = fx.bcx.ins().iconst(types::I8, 0); + for i in 0..16 { + let idx_lane = idx.value_lane(fx, i).load_scalar(fx); + let is_zero = + fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, idx_lane, 16); + let t_idx = fx.bcx.ins().uextend(fx.pointer_type, idx_lane); + let t_lane = t.value_lane_dyn(fx, t_idx).load_scalar(fx); + let res = fx.bcx.ins().select(is_zero, zero, t_lane); + ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); + } + } + + /* + _ if intrinsic.starts_with("llvm.aarch64.neon.sshl.v") + || intrinsic.starts_with("llvm.aarch64.neon.sqshl.v") + // FIXME split this one out once saturating is implemented + || intrinsic.starts_with("llvm.aarch64.neon.sqshlu.v") => + { + intrinsic_args!(fx, args => (a, b); intrinsic); + + simd_pair_for_each_lane(fx, a, b, ret, &|fx, _lane_ty, _res_lane_ty, a, b| { + // FIXME saturate? + fx.bcx.ins().ishl(a, b) + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.sqshrn.v") => { + let (a, imm32) = match args { + [a, imm32] => (a, imm32), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm32 = crate::constant::mir_operand_get_const_val(fx, imm32) + .expect("llvm.aarch64.neon.sqshrn.v* imm32 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm32 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm32 not scalar: {:?}", imm32)) + { + imm32 if imm32 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm32 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.sqshrun.v") => { + let (a, imm32) = match args { + [a, imm32] => (a, imm32), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm32 = crate::constant::mir_operand_get_const_val(fx, imm32) + .expect("llvm.aarch64.neon.sqshrn.v* imm32 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm32 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm32 not scalar: {:?}", imm32)) + { + imm32 if imm32 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm32 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } + */ + _ => { + fx.tcx.dcx().warn(format!( + "unsupported AArch64 llvm intrinsic {}; replacing with trap", + intrinsic + )); + crate::trap::trap_unimplemented(fx, intrinsic); + return; + } + } + + let dest = target.expect("all llvm intrinsics used by stdlib should return"); + let ret_block = fx.get_block(dest); + fx.bcx.ins().jump(ret_block, &[]); +} diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs new file mode 100644 index 00000000000..cb003037c26 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs @@ -0,0 +1,1470 @@ +//! Emulate x86 LLVM intrinsics + +use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_target::asm::*; + +use crate::inline_asm::{codegen_inline_asm_inner, CInlineAsmOperand}; +use crate::intrinsics::*; +use crate::prelude::*; + +pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + intrinsic: &str, + args: &[Spanned<mir::Operand<'tcx>>], + ret: CPlace<'tcx>, + target: Option<BasicBlock>, + span: Span, +) { + match intrinsic { + "llvm.x86.sse2.pause" | "llvm.aarch64.isb" => { + // Spin loop hint + } + + "llvm.x86.avx.vzeroupper" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_zeroupper&ig_expand=7218 + // Do nothing. It is a perf hint anyway. + } + + // Used by is_x86_feature_detected!(); + "llvm.x86.xgetbv" => { + intrinsic_args!(fx, args => (xcr_no); intrinsic); + + let xcr_no = xcr_no.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String( + " + xgetbv + // out = rdx << 32 | rax + shl rdx, 32 + or rax, rdx + " + .into(), + )], + &[ + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)), + value: xcr_no, + }, + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), + late: true, + place: Some(ret), + }, + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), + late: true, + place: None, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sse3.ldu.dq" | "llvm.x86.avx.ldu.dq.256" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_lddqu_si128&ig_expand=4009 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_lddqu_si256&ig_expand=4010 + intrinsic_args!(fx, args => (ptr); intrinsic); + + // FIXME correctly handle unalignedness + let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), ret.layout()); + ret.write_cvalue(fx, val); + } + + "llvm.x86.avx2.gather.d.d" + | "llvm.x86.avx2.gather.d.q" + | "llvm.x86.avx2.gather.d.ps" + | "llvm.x86.avx2.gather.d.pd" + | "llvm.x86.avx2.gather.d.d.256" + | "llvm.x86.avx2.gather.d.q.256" + | "llvm.x86.avx2.gather.d.ps.256" + | "llvm.x86.avx2.gather.d.pd.256" + | "llvm.x86.avx2.gather.q.d" + | "llvm.x86.avx2.gather.q.q" + | "llvm.x86.avx2.gather.q.ps" + | "llvm.x86.avx2.gather.q.pd" + | "llvm.x86.avx2.gather.q.d.256" + | "llvm.x86.avx2.gather.q.q.256" + | "llvm.x86.avx2.gather.q.ps.256" + | "llvm.x86.avx2.gather.q.pd.256" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i64gather_pd&ig_expand=3818 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i64gather_pd&ig_expand=3819 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i64gather_pd&ig_expand=3821 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i64gather_pd&ig_expand=3822 + // ... + + intrinsic_args!(fx, args => (src, ptr, index, mask, scale); intrinsic); + + let (src_lane_count, src_lane_ty) = src.layout().ty.simd_size_and_type(fx.tcx); + let (index_lane_count, index_lane_ty) = index.layout().ty.simd_size_and_type(fx.tcx); + let (mask_lane_count, mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(src_lane_ty, ret_lane_ty); + assert!(index_lane_ty.is_integral()); + assert_eq!(src_lane_count, mask_lane_count); + assert_eq!(src_lane_count, ret_lane_count); + + let lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap(); + let index_lane_clif_ty = fx.clif_type(index_lane_ty).unwrap(); + let mask_lane_clif_ty = fx.clif_type(mask_lane_ty).unwrap(); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + + let ptr = ptr.load_scalar(fx); + let scale = scale.load_scalar(fx); + let scale = fx.bcx.ins().uextend(types::I64, scale); + for lane_idx in 0..std::cmp::min(src_lane_count, index_lane_count) { + let src_lane = src.value_lane(fx, lane_idx).load_scalar(fx); + let index_lane = index.value_lane(fx, lane_idx).load_scalar(fx); + let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx); + let mask_lane = + fx.bcx.ins().bitcast(mask_lane_clif_ty.as_int(), MemFlags::new(), mask_lane); + + let if_enabled = fx.bcx.create_block(); + let if_disabled = fx.bcx.create_block(); + let next = fx.bcx.create_block(); + let res_lane = fx.bcx.append_block_param(next, lane_clif_ty); + + let mask_lane = match mask_lane_clif_ty { + types::I32 | types::F32 => { + fx.bcx.ins().band_imm(mask_lane, 0x8000_0000u64 as i64) + } + types::I64 | types::F64 => { + fx.bcx.ins().band_imm(mask_lane, 0x8000_0000_0000_0000u64 as i64) + } + _ => unreachable!(), + }; + fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]); + fx.bcx.seal_block(if_enabled); + fx.bcx.seal_block(if_disabled); + + fx.bcx.switch_to_block(if_enabled); + let index_lane = if index_lane_clif_ty != types::I64 { + fx.bcx.ins().sextend(types::I64, index_lane) + } else { + index_lane + }; + let offset = fx.bcx.ins().imul(index_lane, scale); + let lane_ptr = fx.bcx.ins().iadd(ptr, offset); + let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), lane_ptr, 0); + fx.bcx.ins().jump(next, &[res]); + + fx.bcx.switch_to_block(if_disabled); + fx.bcx.ins().jump(next, &[src_lane]); + + fx.bcx.seal_block(next); + fx.bcx.switch_to_block(next); + + fx.bcx.ins().nop(); + + ret.place_lane(fx, lane_idx) + .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout)); + } + + for lane_idx in std::cmp::min(src_lane_count, index_lane_count)..ret_lane_count { + let zero_lane = fx.bcx.ins().iconst(mask_lane_clif_ty.as_int(), 0); + let zero_lane = fx.bcx.ins().bitcast(mask_lane_clif_ty, MemFlags::new(), zero_lane); + ret.place_lane(fx, lane_idx) + .write_cvalue(fx, CValue::by_val(zero_lane, ret_lane_layout)); + } + } + + "llvm.x86.sse.max.ps" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_ps&ig_expand=4357 + intrinsic_args!(fx, args => (a, b); intrinsic); + + simd_pair_for_each_lane( + fx, + a, + b, + ret, + &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| fx.bcx.ins().fmax(a_lane, b_lane), + ); + } + + "llvm.x86.sse.min.ps" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_ps&ig_expand=4489 + intrinsic_args!(fx, args => (a, b); intrinsic); + + simd_pair_for_each_lane( + fx, + a, + b, + ret, + &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| fx.bcx.ins().fmin(a_lane, b_lane), + ); + } + + "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => { + let (x, y, kind) = match args { + [x, y, kind] => (x, y, kind), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let x = codegen_operand(fx, &x.node); + let y = codegen_operand(fx, &y.node); + let kind = match &kind.node { + Operand::Constant(const_) => crate::constant::eval_mir_constant(fx, const_).0, + Operand::Copy(_) | Operand::Move(_) => unreachable!("{kind:?}"), + }; + + let flt_cc = match kind + .try_to_bits(Size::from_bytes(1)) + .unwrap_or_else(|| panic!("kind not scalar: {:?}", kind)) + .try_into() + .unwrap() + { + _CMP_EQ_OQ | _CMP_EQ_OS => FloatCC::Equal, + _CMP_LT_OS | _CMP_LT_OQ => FloatCC::LessThan, + _CMP_LE_OS | _CMP_LE_OQ => FloatCC::LessThanOrEqual, + _CMP_UNORD_Q | _CMP_UNORD_S => FloatCC::Unordered, + _CMP_NEQ_UQ | _CMP_NEQ_US => FloatCC::NotEqual, + _CMP_NLT_US | _CMP_NLT_UQ => FloatCC::UnorderedOrGreaterThanOrEqual, + _CMP_NLE_US | _CMP_NLE_UQ => FloatCC::UnorderedOrGreaterThan, + _CMP_ORD_Q | _CMP_ORD_S => FloatCC::Ordered, + _CMP_EQ_UQ | _CMP_EQ_US => FloatCC::UnorderedOrEqual, + _CMP_NGE_US | _CMP_NGE_UQ => FloatCC::UnorderedOrLessThan, + _CMP_NGT_US | _CMP_NGT_UQ => FloatCC::UnorderedOrLessThanOrEqual, + _CMP_FALSE_OQ | _CMP_FALSE_OS => todo!(), + _CMP_NEQ_OQ | _CMP_NEQ_OS => FloatCC::OrderedNotEqual, + _CMP_GE_OS | _CMP_GE_OQ => FloatCC::GreaterThanOrEqual, + _CMP_GT_OS | _CMP_GT_OQ => FloatCC::GreaterThan, + _CMP_TRUE_UQ | _CMP_TRUE_US => todo!(), + + kind => unreachable!("kind {:?}", kind), + }; + + // Copied from stdarch + /// Equal (ordered, non-signaling) + const _CMP_EQ_OQ: i32 = 0x00; + /// Less-than (ordered, signaling) + const _CMP_LT_OS: i32 = 0x01; + /// Less-than-or-equal (ordered, signaling) + const _CMP_LE_OS: i32 = 0x02; + /// Unordered (non-signaling) + const _CMP_UNORD_Q: i32 = 0x03; + /// Not-equal (unordered, non-signaling) + const _CMP_NEQ_UQ: i32 = 0x04; + /// Not-less-than (unordered, signaling) + const _CMP_NLT_US: i32 = 0x05; + /// Not-less-than-or-equal (unordered, signaling) + const _CMP_NLE_US: i32 = 0x06; + /// Ordered (non-signaling) + const _CMP_ORD_Q: i32 = 0x07; + /// Equal (unordered, non-signaling) + const _CMP_EQ_UQ: i32 = 0x08; + /// Not-greater-than-or-equal (unordered, signaling) + const _CMP_NGE_US: i32 = 0x09; + /// Not-greater-than (unordered, signaling) + const _CMP_NGT_US: i32 = 0x0a; + /// False (ordered, non-signaling) + const _CMP_FALSE_OQ: i32 = 0x0b; + /// Not-equal (ordered, non-signaling) + const _CMP_NEQ_OQ: i32 = 0x0c; + /// Greater-than-or-equal (ordered, signaling) + const _CMP_GE_OS: i32 = 0x0d; + /// Greater-than (ordered, signaling) + const _CMP_GT_OS: i32 = 0x0e; + /// True (unordered, non-signaling) + const _CMP_TRUE_UQ: i32 = 0x0f; + /// Equal (ordered, signaling) + const _CMP_EQ_OS: i32 = 0x10; + /// Less-than (ordered, non-signaling) + const _CMP_LT_OQ: i32 = 0x11; + /// Less-than-or-equal (ordered, non-signaling) + const _CMP_LE_OQ: i32 = 0x12; + /// Unordered (signaling) + const _CMP_UNORD_S: i32 = 0x13; + /// Not-equal (unordered, signaling) + const _CMP_NEQ_US: i32 = 0x14; + /// Not-less-than (unordered, non-signaling) + const _CMP_NLT_UQ: i32 = 0x15; + /// Not-less-than-or-equal (unordered, non-signaling) + const _CMP_NLE_UQ: i32 = 0x16; + /// Ordered (signaling) + const _CMP_ORD_S: i32 = 0x17; + /// Equal (unordered, signaling) + const _CMP_EQ_US: i32 = 0x18; + /// Not-greater-than-or-equal (unordered, non-signaling) + const _CMP_NGE_UQ: i32 = 0x19; + /// Not-greater-than (unordered, non-signaling) + const _CMP_NGT_UQ: i32 = 0x1a; + /// False (ordered, signaling) + const _CMP_FALSE_OS: i32 = 0x1b; + /// Not-equal (ordered, signaling) + const _CMP_NEQ_OS: i32 = 0x1c; + /// Greater-than-or-equal (ordered, non-signaling) + const _CMP_GE_OQ: i32 = 0x1d; + /// Greater-than (ordered, non-signaling) + const _CMP_GT_OQ: i32 = 0x1e; + /// True (unordered, signaling) + const _CMP_TRUE_US: i32 = 0x1f; + + simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| { + let res_lane = match lane_ty.kind() { + ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane), + _ => unreachable!("{:?}", lane_ty), + }; + bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane) + }); + } + "llvm.x86.ssse3.pshuf.b.128" | "llvm.x86.avx2.pshuf.b" => { + let (a, b) = match args { + [a, b] => (a, b), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, &a.node); + let b = codegen_operand(fx, &b.node); + + // Based on the pseudocode at https://github.com/rust-lang/stdarch/blob/1cfbca8b38fd9b4282b2f054f61c6ca69fc7ce29/crates/core_arch/src/x86/avx2.rs#L2319-L2332 + let zero = fx.bcx.ins().iconst(types::I8, 0); + for i in 0..16 { + let b_lane = b.value_lane(fx, i).load_scalar(fx); + let is_zero = fx.bcx.ins().band_imm(b_lane, 0x80); + let a_idx = fx.bcx.ins().band_imm(b_lane, 0xf); + let a_idx = fx.bcx.ins().uextend(fx.pointer_type, a_idx); + let a_lane = a.value_lane_dyn(fx, a_idx).load_scalar(fx); + let res = fx.bcx.ins().select(is_zero, zero, a_lane); + ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); + } + + if intrinsic == "llvm.x86.avx2.pshuf.b" { + for i in 16..32 { + let b_lane = b.value_lane(fx, i).load_scalar(fx); + let is_zero = fx.bcx.ins().band_imm(b_lane, 0x80); + let b_lane_masked = fx.bcx.ins().band_imm(b_lane, 0xf); + let a_idx = fx.bcx.ins().iadd_imm(b_lane_masked, 16); + let a_idx = fx.bcx.ins().uextend(fx.pointer_type, a_idx); + let a_lane = a.value_lane_dyn(fx, a_idx).load_scalar(fx); + let res = fx.bcx.ins().select(is_zero, zero, a_lane); + ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted()); + } + } + } + "llvm.x86.avx2.permd" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permutevar8x32_epi32 + intrinsic_args!(fx, args => (a, idx); intrinsic); + + for j in 0..=7 { + let index = idx.value_typed_lane(fx, fx.tcx.types.u32, j).load_scalar(fx); + let index = fx.bcx.ins().uextend(fx.pointer_type, index); + let value = a.value_lane_dyn(fx, index).load_scalar(fx); + ret.place_typed_lane(fx, fx.tcx.types.u32, j).to_ptr().store( + fx, + value, + MemFlags::trusted(), + ); + } + } + "llvm.x86.avx2.vperm2i128" + | "llvm.x86.avx.vperm2f128.ps.256" + | "llvm.x86.avx.vperm2f128.pd.256" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permute2x128_si256 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permute2f128_ps + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permute2f128_pd + let (a, b, imm8) = match args { + [a, b, imm8] => (a, b, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, &a.node); + let b = codegen_operand(fx, &b.node); + let imm8 = codegen_operand(fx, &imm8.node).load_scalar(fx); + + let a_low = a.value_typed_lane(fx, fx.tcx.types.u128, 0).load_scalar(fx); + let a_high = a.value_typed_lane(fx, fx.tcx.types.u128, 1).load_scalar(fx); + + let b_low = b.value_typed_lane(fx, fx.tcx.types.u128, 0).load_scalar(fx); + let b_high = b.value_typed_lane(fx, fx.tcx.types.u128, 1).load_scalar(fx); + + fn select4( + fx: &mut FunctionCx<'_, '_, '_>, + a_high: Value, + a_low: Value, + b_high: Value, + b_low: Value, + control: Value, + ) -> Value { + let a_or_b = fx.bcx.ins().band_imm(control, 0b0010); + let high_or_low = fx.bcx.ins().band_imm(control, 0b0001); + let is_zero = fx.bcx.ins().band_imm(control, 0b1000); + + let zero = fx.bcx.ins().iconst(types::I64, 0); + let zero = fx.bcx.ins().iconcat(zero, zero); + + let res_a = fx.bcx.ins().select(high_or_low, a_high, a_low); + let res_b = fx.bcx.ins().select(high_or_low, b_high, b_low); + let res = fx.bcx.ins().select(a_or_b, res_b, res_a); + fx.bcx.ins().select(is_zero, zero, res) + } + + let control0 = imm8; + let res_low = select4(fx, a_high, a_low, b_high, b_low, control0); + + let control1 = fx.bcx.ins().ushr_imm(imm8, 4); + let res_high = select4(fx, a_high, a_low, b_high, b_low, control1); + + ret.place_typed_lane(fx, fx.tcx.types.u128, 0).to_ptr().store( + fx, + res_low, + MemFlags::trusted(), + ); + ret.place_typed_lane(fx, fx.tcx.types.u128, 1).to_ptr().store( + fx, + res_high, + MemFlags::trusted(), + ); + } + "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => { + intrinsic_args!(fx, args => (a); intrinsic); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().iabs(lane) + }); + } + "llvm.x86.sse2.cvttps2dq" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttps_epi32&ig_expand=2429 + intrinsic_args!(fx, args => (a); intrinsic); + let a = a.load_scalar(fx); + + let value = fx.bcx.ins().x86_cvtt2dq(types::I32X4, a); + let cvalue = CValue::by_val(value, ret.layout()); + ret.write_cvalue(fx, cvalue); + } + "llvm.x86.sse2.cvtps2dq" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtps_epi32 + intrinsic_args!(fx, args => (a); intrinsic); + let a = a.load_scalar(fx); + + // Using inline asm instead of fcvt_to_sint_sat as unrepresentable values are turned + // into 0x80000000 for which Cranelift doesn't have a native instruction. + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("cvtps2dq xmm0, xmm0".into())], + &[CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + "llvm.x86.addcarry.32" | "llvm.x86.addcarry.64" => { + intrinsic_args!(fx, args => (c_in, a, b); intrinsic); + let c_in = c_in.load_scalar(fx); + + let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b); + + let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty])); + let val = CValue::by_val_pair(cb_out, c, layout); + ret.write_cvalue(fx, val); + } + "llvm.x86.addcarryx.u32" | "llvm.x86.addcarryx.u64" => { + intrinsic_args!(fx, args => (c_in, a, b, out); intrinsic); + let c_in = c_in.load_scalar(fx); + + let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b); + + Pointer::new(out.load_scalar(fx)).store(fx, c, MemFlags::trusted()); + ret.write_cvalue(fx, CValue::by_val(cb_out, fx.layout_of(fx.tcx.types.u8))); + } + "llvm.x86.subborrow.32" | "llvm.x86.subborrow.64" => { + intrinsic_args!(fx, args => (b_in, a, b); intrinsic); + let b_in = b_in.load_scalar(fx); + + let (cb_out, c) = llvm_add_sub(fx, BinOp::Sub, b_in, a, b); + + let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty])); + let val = CValue::by_val_pair(cb_out, c, layout); + ret.write_cvalue(fx, val); + } + "llvm.x86.sse2.pavg.b" | "llvm.x86.sse2.pavg.w" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + // FIXME use vector instructions when possible + simd_pair_for_each_lane( + fx, + a, + b, + ret, + &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| { + // (a + b + 1) >> 1 + let lane_ty = fx.bcx.func.dfg.value_type(a_lane); + let a_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), a_lane); + let b_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), b_lane); + let sum = fx.bcx.ins().iadd(a_lane, b_lane); + let num_plus_one = fx.bcx.ins().iadd_imm(sum, 1); + let res = fx.bcx.ins().ushr_imm(num_plus_one, 1); + fx.bcx.ins().ireduce(lane_ty, res) + }, + ); + } + "llvm.x86.sse2.psra.w" => { + intrinsic_args!(fx, args => (a, count); intrinsic); + + let count_lane = count.force_stack(fx).0.load(fx, types::I64, MemFlags::trusted()); + let lane_ty = fx.clif_type(a.layout().ty.simd_size_and_type(fx.tcx).1).unwrap(); + let max_count = fx.bcx.ins().iconst(types::I64, i64::from(lane_ty.bits() - 1)); + let saturated_count = fx.bcx.ins().umin(count_lane, max_count); + + // FIXME use vector instructions when possible + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, a_lane| { + fx.bcx.ins().sshr(a_lane, saturated_count) + }); + } + "llvm.x86.sse2.psad.bw" | "llvm.x86.avx2.psad.bw" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sad_epu8&ig_expand=5770 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_sad_epu8&ig_expand=5771 + intrinsic_args!(fx, args => (a, b); intrinsic); + + assert_eq!(a.layout(), b.layout()); + let layout = a.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(lane_ty, fx.tcx.types.u8); + assert_eq!(ret_lane_ty, fx.tcx.types.u64); + assert_eq!(lane_count, ret_lane_count * 8); + + let ret_lane_layout = fx.layout_of(fx.tcx.types.u64); + for out_lane_idx in 0..lane_count / 8 { + let mut lane_diff_acc = fx.bcx.ins().iconst(types::I64, 0); + + for lane_idx in out_lane_idx * 8..out_lane_idx * 8 + 8 { + let a_lane = a.value_lane(fx, lane_idx).load_scalar(fx); + let a_lane = fx.bcx.ins().uextend(types::I16, a_lane); + let b_lane = b.value_lane(fx, lane_idx).load_scalar(fx); + let b_lane = fx.bcx.ins().uextend(types::I16, b_lane); + + let lane_diff = fx.bcx.ins().isub(a_lane, b_lane); + let abs_lane_diff = fx.bcx.ins().iabs(lane_diff); + let abs_lane_diff = fx.bcx.ins().uextend(types::I64, abs_lane_diff); + lane_diff_acc = fx.bcx.ins().iadd(lane_diff_acc, abs_lane_diff); + } + + let res_lane = CValue::by_val(lane_diff_acc, ret_lane_layout); + + ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane); + } + } + "llvm.x86.ssse3.pmadd.ub.sw.128" | "llvm.x86.avx2.pmadd.ub.sw" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maddubs_epi16&ig_expand=4267 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_maddubs_epi16&ig_expand=4270 + intrinsic_args!(fx, args => (a, b); intrinsic); + + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(lane_ty, fx.tcx.types.u8); + assert_eq!(ret_lane_ty, fx.tcx.types.i16); + assert_eq!(lane_count, ret_lane_count * 2); + + let ret_lane_layout = fx.layout_of(fx.tcx.types.i16); + for out_lane_idx in 0..lane_count / 2 { + let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx); + let a_lane0 = fx.bcx.ins().uextend(types::I16, a_lane0); + let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx); + let b_lane0 = fx.bcx.ins().sextend(types::I16, b_lane0); + + let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx); + let a_lane1 = fx.bcx.ins().uextend(types::I16, a_lane1); + let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx); + let b_lane1 = fx.bcx.ins().sextend(types::I16, b_lane1); + + let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0); + let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1); + + let (val, has_overflow) = fx.bcx.ins().sadd_overflow(mul0, mul1); + + let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, mul1, 0); + + let min = fx.bcx.ins().iconst(types::I16, i64::from(i16::MIN as u16)); + let max = fx.bcx.ins().iconst(types::I16, i64::from(i16::MAX as u16)); + + let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); + let res_lane = fx.bcx.ins().select(has_overflow, sat_val, val); + + let res_lane = CValue::by_val(res_lane, ret_lane_layout); + + ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane); + } + } + "llvm.x86.sse2.pmadd.wd" | "llvm.x86.avx2.pmadd.wd" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_madd_epi16&ig_expand=4231 + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_madd_epi16&ig_expand=4234 + intrinsic_args!(fx, args => (a, b); intrinsic); + + assert_eq!(a.layout(), b.layout()); + let layout = a.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(lane_ty, fx.tcx.types.i16); + assert_eq!(ret_lane_ty, fx.tcx.types.i32); + assert_eq!(lane_count, ret_lane_count * 2); + + let ret_lane_layout = fx.layout_of(fx.tcx.types.i32); + for out_lane_idx in 0..lane_count / 2 { + let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx); + let a_lane0 = fx.bcx.ins().sextend(types::I32, a_lane0); + let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx); + let b_lane0 = fx.bcx.ins().sextend(types::I32, b_lane0); + + let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx); + let a_lane1 = fx.bcx.ins().sextend(types::I32, a_lane1); + let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx); + let b_lane1 = fx.bcx.ins().sextend(types::I32, b_lane1); + + let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0); + let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1); + + let res_lane = fx.bcx.ins().iadd(mul0, mul1); + let res_lane = CValue::by_val(res_lane, ret_lane_layout); + + ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane); + } + } + + "llvm.x86.ssse3.pmul.hr.sw.128" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mulhrs_epi16&ig_expand=4782 + intrinsic_args!(fx, args => (a, b); intrinsic); + + assert_eq!(a.layout(), b.layout()); + let layout = a.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(lane_ty, fx.tcx.types.i16); + assert_eq!(ret_lane_ty, fx.tcx.types.i16); + assert_eq!(lane_count, ret_lane_count); + + let ret_lane_layout = fx.layout_of(fx.tcx.types.i16); + for out_lane_idx in 0..lane_count { + let a_lane = a.value_lane(fx, out_lane_idx).load_scalar(fx); + let a_lane = fx.bcx.ins().sextend(types::I32, a_lane); + let b_lane = b.value_lane(fx, out_lane_idx).load_scalar(fx); + let b_lane = fx.bcx.ins().sextend(types::I32, b_lane); + + let mul: Value = fx.bcx.ins().imul(a_lane, b_lane); + let shifted = fx.bcx.ins().ushr_imm(mul, 14); + let incremented = fx.bcx.ins().iadd_imm(shifted, 1); + let shifted_again = fx.bcx.ins().ushr_imm(incremented, 1); + + let res_lane = fx.bcx.ins().ireduce(types::I16, shifted_again); + let res_lane = CValue::by_val(res_lane, ret_lane_layout); + + ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane); + } + } + + "llvm.x86.sse2.packuswb.128" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packus_epi16&ig_expand=4903 + intrinsic_args!(fx, args => (a, b); intrinsic); + + pack_instruction(fx, a, b, ret, PackSize::U8, PackWidth::Sse); + } + + "llvm.x86.sse2.packsswb.128" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packs_epi16&ig_expand=4848 + intrinsic_args!(fx, args => (a, b); intrinsic); + + pack_instruction(fx, a, b, ret, PackSize::S8, PackWidth::Sse); + } + + "llvm.x86.avx2.packuswb" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packus_epi16&ig_expand=4906 + intrinsic_args!(fx, args => (a, b); intrinsic); + + pack_instruction(fx, a, b, ret, PackSize::U8, PackWidth::Avx); + } + + "llvm.x86.avx2.packsswb" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packs_epi16&ig_expand=4851 + intrinsic_args!(fx, args => (a, b); intrinsic); + + pack_instruction(fx, a, b, ret, PackSize::S8, PackWidth::Avx); + } + + "llvm.x86.sse41.packusdw" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packus_epi32&ig_expand=4912 + intrinsic_args!(fx, args => (a, b); intrinsic); + + pack_instruction(fx, a, b, ret, PackSize::U16, PackWidth::Sse); + } + + "llvm.x86.sse2.packssdw.128" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packs_epi32&ig_expand=4889 + intrinsic_args!(fx, args => (a, b); intrinsic); + + pack_instruction(fx, a, b, ret, PackSize::S16, PackWidth::Sse); + } + + "llvm.x86.avx2.packusdw" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packus_epi32&ig_expand=4883 + intrinsic_args!(fx, args => (a, b); intrinsic); + + pack_instruction(fx, a, b, ret, PackSize::U16, PackWidth::Avx); + } + + "llvm.x86.avx2.packssdw" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_packs_epi32&ig_expand=4892 + intrinsic_args!(fx, args => (a, b); intrinsic); + + pack_instruction(fx, a, b, ret, PackSize::S16, PackWidth::Avx); + } + + "llvm.x86.sse42.crc32.32.8" + | "llvm.x86.sse42.crc32.32.16" + | "llvm.x86.sse42.crc32.32.32" + | "llvm.x86.sse42.crc32.64.64" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#ig_expand=1419&text=_mm_crc32_u32 + intrinsic_args!(fx, args => (crc, v); intrinsic); + + let crc = crc.load_scalar(fx); + let v = v.load_scalar(fx); + + let asm = match intrinsic { + "llvm.x86.sse42.crc32.32.8" => "crc32 eax, dl", + "llvm.x86.sse42.crc32.32.16" => "crc32 eax, dx", + "llvm.x86.sse42.crc32.32.32" => "crc32 eax, edx", + "llvm.x86.sse42.crc32.64.64" => "crc32 rax, rdx", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), + _late: true, + in_value: crc, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), + value: v, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sse42.pcmpestri128" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestri&ig_expand=939 + intrinsic_args!(fx, args => (a, la, b, lb, _imm8); intrinsic); + + let a = a.load_scalar(fx); + let la = la.load_scalar(fx); + let b = b.load_scalar(fx); + let lb = lb.load_scalar(fx); + + let imm8 = + if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[4].node) { + imm8 + } else { + fx.tcx + .dcx() + .span_fatal(span, "Index argument for `_mm_cmpestri` is not a constant"); + }; + + let imm8 = imm8.to_u8(); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(format!("pcmpestri xmm0, xmm1, {imm8}").into())], + &[ + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + value: a, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + value: b, + }, + // Implicit argument to the pcmpestri intrinsic + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), + value: la, + }, + // Implicit argument to the pcmpestri intrinsic + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), + value: lb, + }, + // Implicit result of the pcmpestri intrinsic + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)), + late: true, + place: Some(ret), + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sse42.pcmpestrm128" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestrm&ig_expand=940 + intrinsic_args!(fx, args => (a, la, b, lb, _imm8); intrinsic); + + let a = a.load_scalar(fx); + let la = la.load_scalar(fx); + let b = b.load_scalar(fx); + let lb = lb.load_scalar(fx); + + let imm8 = + if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[4].node) { + imm8 + } else { + fx.tcx + .dcx() + .span_fatal(span, "Index argument for `_mm_cmpestrm` is not a constant"); + }; + + let imm8 = imm8.to_u8(); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(format!("pcmpestrm xmm0, xmm1, {imm8}").into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + value: b, + }, + // Implicit argument to the pcmpestri intrinsic + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), + value: la, + }, + // Implicit argument to the pcmpestri intrinsic + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), + value: lb, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.pclmulqdq" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_clmulepi64_si128&ig_expand=772 + intrinsic_args!(fx, args => (a, b, _imm8); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let imm8 = + if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[2].node) { + imm8 + } else { + fx.tcx.dcx().span_fatal( + span, + "Index argument for `_mm_clmulepi64_si128` is not a constant", + ); + }; + + let imm8 = imm8.to_u8(); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(format!("pclmulqdq xmm0, xmm1, {imm8}").into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.aesni.aeskeygenassist" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aeskeygenassist_si128&ig_expand=261 + intrinsic_args!(fx, args => (a, _imm8); intrinsic); + + let a = a.load_scalar(fx); + + let imm8 = + if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[1].node) { + imm8 + } else { + fx.tcx.dcx().span_fatal( + span, + "Index argument for `_mm_aeskeygenassist_si128` is not a constant", + ); + }; + + let imm8 = imm8.to_u8(); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String( + format!("aeskeygenassist xmm0, xmm0, {imm8}").into(), + )], + &[CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.aesni.aesimc" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesimc_si128&ig_expand=260 + intrinsic_args!(fx, args => (a); intrinsic); + + let a = a.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("aesimc xmm0, xmm0".into())], + &[CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.aesni.aesenc" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenc_si128&ig_expand=252 + intrinsic_args!(fx, args => (a, round_key); intrinsic); + + let a = a.load_scalar(fx); + let round_key = round_key.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("aesenc xmm0, xmm1".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + value: round_key, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.aesni.aesenclast" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenclast_si128&ig_expand=257 + intrinsic_args!(fx, args => (a, round_key); intrinsic); + + let a = a.load_scalar(fx); + let round_key = round_key.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("aesenclast xmm0, xmm1".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + value: round_key, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.aesni.aesdec" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec_si128&ig_expand=242 + intrinsic_args!(fx, args => (a, round_key); intrinsic); + + let a = a.load_scalar(fx); + let round_key = round_key.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("aesdec xmm0, xmm1".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + value: round_key, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.aesni.aesdeclast" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdeclast_si128&ig_expand=247 + intrinsic_args!(fx, args => (a, round_key); intrinsic); + + let a = a.load_scalar(fx); + let round_key = round_key.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("aesdeclast xmm0, xmm1".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + value: round_key, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sha1rnds4" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1rnds4_epu32&ig_expand=5877 + intrinsic_args!(fx, args => (a, b, _func); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let func = if let Some(func) = + crate::constant::mir_operand_get_const_val(fx, &args[2].node) + { + func + } else { + fx.tcx + .dcx() + .span_fatal(span, "Func argument for `_mm_sha1rnds4_epu32` is not a constant"); + }; + + let func = func.to_u8(); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(format!("sha1rnds4 xmm1, xmm2, {func}").into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sha1msg1" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1msg1_epu32&ig_expand=5874 + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha1msg1 xmm1, xmm2".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sha1msg2" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1msg2_epu32&ig_expand=5875 + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha1msg2 xmm1, xmm2".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sha1nexte" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1nexte_epu32&ig_expand=5876 + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha1nexte xmm1, xmm2".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sha256rnds2" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256rnds2_epu32&ig_expand=5977 + intrinsic_args!(fx, args => (a, b, k); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let k = k.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha256rnds2 xmm1, xmm2".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), + value: b, + }, + // Implicit argument to the sha256rnds2 instruction + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + value: k, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sha256msg1" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256msg1_epu32&ig_expand=5975 + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha256msg1 xmm1, xmm2".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.sha256msg2" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256msg2_epu32&ig_expand=5976 + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha256msg2 xmm1, xmm2".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.x86.avx.ptestz.256" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_testz_si256&ig_expand=6945 + intrinsic_args!(fx, args => (a, b); intrinsic); + + assert_eq!(a.layout(), b.layout()); + let layout = a.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + assert_eq!(lane_ty, fx.tcx.types.i64); + assert_eq!(ret.layout().ty, fx.tcx.types.i32); + assert_eq!(lane_count, 4); + + let a_lane0 = a.value_lane(fx, 0).load_scalar(fx); + let a_lane1 = a.value_lane(fx, 1).load_scalar(fx); + let a_lane2 = a.value_lane(fx, 2).load_scalar(fx); + let a_lane3 = a.value_lane(fx, 3).load_scalar(fx); + let b_lane0 = b.value_lane(fx, 0).load_scalar(fx); + let b_lane1 = b.value_lane(fx, 1).load_scalar(fx); + let b_lane2 = b.value_lane(fx, 2).load_scalar(fx); + let b_lane3 = b.value_lane(fx, 3).load_scalar(fx); + + let zero0 = fx.bcx.ins().band(a_lane0, b_lane0); + let zero1 = fx.bcx.ins().band(a_lane1, b_lane1); + let zero2 = fx.bcx.ins().band(a_lane2, b_lane2); + let zero3 = fx.bcx.ins().band(a_lane3, b_lane3); + + let all_zero0 = fx.bcx.ins().bor(zero0, zero1); + let all_zero1 = fx.bcx.ins().bor(zero2, zero3); + let all_zero = fx.bcx.ins().bor(all_zero0, all_zero1); + + let res = fx.bcx.ins().icmp_imm(IntCC::Equal, all_zero, 0); + let res = CValue::by_val( + fx.bcx.ins().uextend(types::I32, res), + fx.layout_of(fx.tcx.types.i32), + ); + ret.write_cvalue(fx, res); + } + + "llvm.x86.rdtsc" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_rdtsc&ig_expand=5273 + + let res_place = CPlace::new_stack_slot( + fx, + fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u32, fx.tcx.types.u32])), + ); + let eax_place = res_place.place_field(fx, FieldIdx::new(0)); + let edx_place = res_place.place_field(fx, FieldIdx::new(1)); + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("rdtsc".into())], + &[ + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), + late: true, + place: Some(eax_place), + }, + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), + late: true, + place: Some(edx_place), + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::NOMEM, + ); + let res = res_place.to_cvalue(fx); + ret.write_cvalue_transmute(fx, res); + } + + _ => { + fx.tcx + .dcx() + .warn(format!("unsupported x86 llvm intrinsic {}; replacing with trap", intrinsic)); + crate::trap::trap_unimplemented(fx, intrinsic); + return; + } + } + + let dest = target.expect("all llvm intrinsics used by stdlib should return"); + let ret_block = fx.get_block(dest); + fx.bcx.ins().jump(ret_block, &[]); +} + +// llvm.x86.avx2.vperm2i128 +// llvm.x86.ssse3.pshuf.b.128 +// llvm.x86.avx2.pshuf.b + +fn llvm_add_sub<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + cb_in: Value, + a: CValue<'tcx>, + b: CValue<'tcx>, +) -> (Value, Value) { + assert_eq!(a.layout().ty, b.layout().ty); + + // c + carry -> c + first intermediate carry or borrow respectively + let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b); + let c = int0.value_field(fx, FieldIdx::ZERO); + let cb0 = int0.value_field(fx, FieldIdx::new(1)).load_scalar(fx); + + // c + carry -> c + second intermediate carry or borrow respectively + let clif_ty = fx.clif_type(a.layout().ty).unwrap(); + let cb_in_as_int = fx.bcx.ins().uextend(clif_ty, cb_in); + let cb_in_as_int = CValue::by_val(cb_in_as_int, fx.layout_of(a.layout().ty)); + let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_int); + let (c, cb1) = int1.load_scalar_pair(fx); + + // carry0 | carry1 -> carry or borrow respectively + let cb_out = fx.bcx.ins().bor(cb0, cb1); + + (cb_out, c) +} + +enum PackSize { + U8, + U16, + S8, + S16, +} + +impl PackSize { + fn ret_clif_type(&self) -> Type { + match self { + Self::U8 | Self::S8 => types::I8, + Self::U16 | Self::S16 => types::I16, + } + } + fn src_clif_type(&self) -> Type { + match self { + Self::U8 | Self::S8 => types::I16, + Self::U16 | Self::S16 => types::I32, + } + } + fn src_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> { + match self { + Self::U8 | Self::S8 => tcx.types.i16, + Self::U16 | Self::S16 => tcx.types.i32, + } + } + fn ret_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> { + match self { + Self::U8 => tcx.types.u8, + Self::S8 => tcx.types.i8, + Self::U16 => tcx.types.u16, + Self::S16 => tcx.types.i16, + } + } + fn max(&self) -> i64 { + match self { + Self::U8 => u8::MAX as u64 as i64, + Self::S8 => i8::MAX as u8 as u64 as i64, + Self::U16 => u16::MAX as u64 as i64, + Self::S16 => i16::MAX as u64 as u64 as i64, + } + } + fn min(&self) -> i64 { + match self { + Self::U8 | Self::U16 => 0, + Self::S8 => i16::from(i8::MIN) as u16 as i64, + Self::S16 => i32::from(i16::MIN) as u32 as i64, + } + } +} + +enum PackWidth { + Sse = 1, + Avx = 2, +} +impl PackWidth { + fn divisor(&self) -> u64 { + match self { + Self::Sse => 1, + Self::Avx => 2, + } + } +} + +/// Implement an x86 pack instruction with the intrinsic `_mm{,256}pack{us,s}_epi{16,32}`. +/// Validated for correctness against LLVM, see commit `c8f5d35508e062bd2d95e6c03429bfec831db6d3`. +fn pack_instruction<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + a: CValue<'tcx>, + b: CValue<'tcx>, + ret: CPlace<'tcx>, + ret_size: PackSize, + width: PackWidth, +) { + assert_eq!(a.layout(), b.layout()); + let layout = a.layout(); + + let (src_lane_count, src_lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(src_lane_ty, ret_size.src_ty(fx.tcx)); + assert_eq!(ret_lane_ty, ret_size.ret_ty(fx.tcx)); + assert_eq!(src_lane_count * 2, ret_lane_count); + + let min = fx.bcx.ins().iconst(ret_size.src_clif_type(), ret_size.min()); + let max = fx.bcx.ins().iconst(ret_size.src_clif_type(), ret_size.max()); + let ret_lane_layout = fx.layout_of(ret_size.ret_ty(fx.tcx)); + + let mut round = |source: CValue<'tcx>, source_offset: u64, dest_offset: u64| { + let step_amount = src_lane_count / width.divisor(); + let dest_offset = step_amount * dest_offset; + for idx in 0..step_amount { + let lane = source.value_lane(fx, step_amount * source_offset + idx).load_scalar(fx); + let sat = fx.bcx.ins().smax(lane, min); + let sat = match ret_size { + PackSize::U8 | PackSize::U16 => fx.bcx.ins().umin(sat, max), + PackSize::S8 | PackSize::S16 => fx.bcx.ins().smin(sat, max), + }; + let res = fx.bcx.ins().ireduce(ret_size.ret_clif_type(), sat); + let res_lane = CValue::by_val(res, ret_lane_layout); + ret.place_lane(fx, dest_offset + idx).write_cvalue(fx, res_lane); + } + }; + + round(a, 0, 0); + round(b, 0, 1); + + if let PackWidth::Avx = width { + round(a, 1, 2); + round(b, 1, 3); + } +} diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs new file mode 100644 index 00000000000..b21c559e668 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -0,0 +1,1281 @@ +//! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"` +//! and LLVM intrinsics that have symbol names starting with `llvm.`. + +macro_rules! intrinsic_args { + ($fx:expr, $args:expr => ($($arg:tt),*); $intrinsic:expr) => { + #[allow(unused_parens)] + let ($($arg),*) = if let [$($arg),*] = $args { + ($(codegen_operand($fx, &($arg).node)),*) + } else { + $crate::intrinsics::bug_on_incorrect_arg_count($intrinsic); + }; + } +} + +mod llvm; +mod llvm_aarch64; +mod llvm_x86; +mod simd; + +use cranelift_codegen::ir::AtomicRmwOp; +use rustc_middle::ty; +use rustc_middle::ty::layout::{HasParamEnv, ValidityRequirement}; +use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; +use rustc_middle::ty::GenericArgsRef; +use rustc_span::source_map::Spanned; +use rustc_span::symbol::{sym, Symbol}; + +pub(crate) use self::llvm::codegen_llvm_intrinsic_call; +use crate::cast::clif_intcast; +use crate::prelude::*; + +fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! { + bug!("wrong number of args for intrinsic {}", intrinsic); +} + +fn report_atomic_type_validation_error<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + intrinsic: Symbol, + span: Span, + ty: Ty<'tcx>, +) { + fx.tcx.dcx().span_err( + span, + format!( + "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`", + intrinsic, ty + ), + ); + // Prevent verifier error + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); +} + +pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Type { + let (element, count) = match layout.abi { + Abi::Vector { element, count } => (element, count), + _ => unreachable!(), + }; + + scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()).unwrap() +} + +fn simd_for_each_lane<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + val: CValue<'tcx>, + ret: CPlace<'tcx>, + f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value) -> Value, +) { + let layout = val.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + assert_eq!(lane_count, ret_lane_count); + + for lane_idx in 0..lane_count { + let lane = val.value_lane(fx, lane_idx).load_scalar(fx); + + let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lane); + let res_lane = CValue::by_val(res_lane, ret_lane_layout); + + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } +} + +fn simd_pair_for_each_lane_typed<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + x: CValue<'tcx>, + y: CValue<'tcx>, + ret: CPlace<'tcx>, + f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, CValue<'tcx>, CValue<'tcx>) -> CValue<'tcx>, +) { + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, _ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(lane_count, ret_lane_count); + + for lane_idx in 0..lane_count { + let x_lane = x.value_lane(fx, lane_idx); + let y_lane = y.value_lane(fx, lane_idx); + + let res_lane = f(fx, x_lane, y_lane); + + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } +} + +fn simd_pair_for_each_lane<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + x: CValue<'tcx>, + y: CValue<'tcx>, + ret: CPlace<'tcx>, + f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value, +) { + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + assert_eq!(lane_count, ret_lane_count); + + for lane_idx in 0..lane_count { + let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx); + let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx); + + let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane); + let res_lane = CValue::by_val(res_lane, ret_lane_layout); + + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } +} + +fn simd_horizontal_pair_for_each_lane<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + x: CValue<'tcx>, + y: CValue<'tcx>, + ret: CPlace<'tcx>, + f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value, +) { + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + assert_eq!(lane_count, ret_lane_count); + + for lane_idx in 0..lane_count { + let src = if lane_idx < (lane_count / 2) { x } else { y }; + let src_idx = lane_idx % (lane_count / 2); + + let lhs_lane = src.value_lane(fx, src_idx * 2).load_scalar(fx); + let rhs_lane = src.value_lane(fx, src_idx * 2 + 1).load_scalar(fx); + + let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lhs_lane, rhs_lane); + let res_lane = CValue::by_val(res_lane, ret_lane_layout); + + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } +} + +fn simd_trio_for_each_lane<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + x: CValue<'tcx>, + y: CValue<'tcx>, + z: CValue<'tcx>, + ret: CPlace<'tcx>, + f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value, Value) -> Value, +) { + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + assert_eq!(lane_count, ret_lane_count); + + for lane_idx in 0..lane_count { + let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx); + let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx); + let z_lane = z.value_lane(fx, lane_idx).load_scalar(fx); + + let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane, z_lane); + let res_lane = CValue::by_val(res_lane, ret_lane_layout); + + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } +} + +fn simd_reduce<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + val: CValue<'tcx>, + acc: Option<Value>, + ret: CPlace<'tcx>, + f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Value, Value) -> Value, +) { + let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + assert_eq!(lane_layout, ret.layout()); + + let (mut res_val, start_lane) = + if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) }; + for lane_idx in start_lane..lane_count { + let lane = val.value_lane(fx, lane_idx).load_scalar(fx); + res_val = f(fx, lane_layout.ty, res_val, lane); + } + let res = CValue::by_val(res_val, lane_layout); + ret.write_cvalue(fx, res); +} + +// FIXME move all uses to `simd_reduce` +fn simd_reduce_bool<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + val: CValue<'tcx>, + ret: CPlace<'tcx>, + f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value, +) { + let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + assert!(ret.layout().ty.is_bool()); + + let res_val = val.value_lane(fx, 0).load_scalar(fx); + let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean + for lane_idx in 1..lane_count { + let lane = val.value_lane(fx, lane_idx).load_scalar(fx); + let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean + res_val = f(fx, res_val, lane); + } + let res_val = if fx.bcx.func.dfg.value_type(res_val) != types::I8 { + fx.bcx.ins().ireduce(types::I8, res_val) + } else { + res_val + }; + let res = CValue::by_val(res_val, ret.layout()); + ret.write_cvalue(fx, res); +} + +fn bool_to_zero_or_max_uint<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + ty: Ty<'tcx>, + val: Value, +) -> Value { + let ty = fx.clif_type(ty).unwrap(); + + let int_ty = match ty { + types::F32 => types::I32, + types::F64 => types::I64, + ty => ty, + }; + + let mut res = fx.bcx.ins().bmask(int_ty, val); + + if ty.is_float() { + res = codegen_bitcast(fx, ty, res); + } + + res +} + +pub(crate) fn codegen_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + instance: Instance<'tcx>, + args: &[Spanned<mir::Operand<'tcx>>], + destination: CPlace<'tcx>, + target: Option<BasicBlock>, + source_info: mir::SourceInfo, +) -> Result<(), Instance<'tcx>> { + let intrinsic = fx.tcx.item_name(instance.def_id()); + let instance_args = instance.args; + + if intrinsic.as_str().starts_with("simd_") { + self::simd::codegen_simd_intrinsic_call( + fx, + intrinsic, + instance_args, + args, + destination, + target.expect("target for simd intrinsic"), + source_info.span, + ); + } else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) { + let ret_block = fx.get_block(target.expect("target for float intrinsic")); + fx.bcx.ins().jump(ret_block, &[]); + } else { + codegen_regular_intrinsic_call( + fx, + instance, + intrinsic, + instance_args, + args, + destination, + target, + source_info, + )?; + } + Ok(()) +} + +fn codegen_float_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + intrinsic: Symbol, + args: &[Spanned<mir::Operand<'tcx>>], + ret: CPlace<'tcx>, +) -> bool { + let (name, arg_count, ty, clif_ty) = match intrinsic { + sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32), + sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64), + sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32), + sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64), + sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32), + sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64), + sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins + sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins + sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32), + sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64), + sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32), + sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64), + sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32), + sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64), + sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32), + sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64), + sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32, types::F32), + sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64, types::F64), + sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), + sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64), + sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32), + sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64), + sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32), + sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64), + sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32), + sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64), + sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32), + sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64), + sym::rintf32 => ("rintf", 1, fx.tcx.types.f32, types::F32), + sym::rintf64 => ("rint", 1, fx.tcx.types.f64, types::F64), + sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32), + sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64), + sym::roundevenf32 => ("roundevenf", 1, fx.tcx.types.f32, types::F32), + sym::roundevenf64 => ("roundeven", 1, fx.tcx.types.f64, types::F64), + sym::nearbyintf32 => ("nearbyintf", 1, fx.tcx.types.f32, types::F32), + sym::nearbyintf64 => ("nearbyint", 1, fx.tcx.types.f64, types::F64), + sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32), + sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64), + sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32), + sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64), + _ => return false, + }; + + if args.len() != arg_count { + bug!("wrong number of args for intrinsic {:?}", intrinsic); + } + + let (a, b, c); + let args = match args { + [x] => { + a = [codegen_operand(fx, &x.node).load_scalar(fx)]; + &a as &[_] + } + [x, y] => { + b = [ + codegen_operand(fx, &x.node).load_scalar(fx), + codegen_operand(fx, &y.node).load_scalar(fx), + ]; + &b + } + [x, y, z] => { + c = [ + codegen_operand(fx, &x.node).load_scalar(fx), + codegen_operand(fx, &y.node).load_scalar(fx), + codegen_operand(fx, &z.node).load_scalar(fx), + ]; + &c + } + _ => unreachable!(), + }; + + let layout = fx.layout_of(ty); + let res = match intrinsic { + sym::fmaf32 | sym::fmaf64 => { + CValue::by_val(fx.bcx.ins().fma(args[0], args[1], args[2]), layout) + } + sym::copysignf32 | sym::copysignf64 => { + CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout) + } + sym::fabsf32 + | sym::fabsf64 + | sym::floorf32 + | sym::floorf64 + | sym::ceilf32 + | sym::ceilf64 + | sym::truncf32 + | sym::truncf64 + | sym::nearbyintf32 + | sym::nearbyintf64 + | sym::sqrtf32 + | sym::sqrtf64 => { + let val = match intrinsic { + sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(args[0]), + sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]), + sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]), + sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]), + sym::nearbyintf32 | sym::nearbyintf64 => fx.bcx.ins().nearest(args[0]), + sym::sqrtf32 | sym::sqrtf64 => fx.bcx.ins().sqrt(args[0]), + _ => unreachable!(), + }; + + CValue::by_val(val, layout) + } + + // These intrinsics aren't supported natively by Cranelift. + // Lower them to a libcall. + sym::powif32 | sym::powif64 => { + let input_tys: Vec<_> = vec![AbiParam::new(clif_ty), AbiParam::new(types::I32)]; + let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0]; + CValue::by_val(ret_val, fx.layout_of(ty)) + } + _ => { + let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect(); + let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0]; + CValue::by_val(ret_val, fx.layout_of(ty)) + } + }; + + ret.write_cvalue(fx, res); + + true +} + +fn codegen_regular_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + instance: Instance<'tcx>, + intrinsic: Symbol, + generic_args: GenericArgsRef<'tcx>, + args: &[Spanned<mir::Operand<'tcx>>], + ret: CPlace<'tcx>, + destination: Option<BasicBlock>, + source_info: mir::SourceInfo, +) -> Result<(), Instance<'tcx>> { + assert_eq!(generic_args, instance.args); + let usize_layout = fx.layout_of(fx.tcx.types.usize); + + match intrinsic { + sym::abort => { + fx.bcx.ins().trap(TrapCode::User(0)); + return Ok(()); + } + sym::likely | sym::unlikely => { + intrinsic_args!(fx, args => (a); intrinsic); + + ret.write_cvalue(fx, a); + } + sym::breakpoint => { + intrinsic_args!(fx, args => (); intrinsic); + + fx.bcx.ins().debugtrap(); + } + sym::copy => { + intrinsic_args!(fx, args => (src, dst, count); intrinsic); + let src = src.load_scalar(fx); + let dst = dst.load_scalar(fx); + let count = count.load_scalar(fx); + + let elem_ty = generic_args.type_at(0); + let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); + assert_eq!(args.len(), 3); + let byte_amount = + if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; + + // FIXME emit_small_memmove + fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount); + } + sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => { + // NOTE: the volatile variants have src and dst swapped + intrinsic_args!(fx, args => (dst, src, count); intrinsic); + let dst = dst.load_scalar(fx); + let src = src.load_scalar(fx); + let count = count.load_scalar(fx); + + let elem_ty = generic_args.type_at(0); + let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); + assert_eq!(args.len(), 3); + let byte_amount = + if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; + + // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} + if intrinsic == sym::volatile_copy_nonoverlapping_memory { + // FIXME emit_small_memcpy + fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount); + } else { + // FIXME emit_small_memmove + fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount); + } + } + sym::size_of_val => { + intrinsic_args!(fx, args => (ptr); intrinsic); + + let layout = fx.layout_of(generic_args.type_at(0)); + // Note: Can't use is_unsized here as truly unsized types need to take the fixed size + // branch + let meta = if let Abi::ScalarPair(_, _) = ptr.layout().abi { + Some(ptr.load_scalar_pair(fx).1) + } else { + None + }; + let (size, _align) = crate::unsize::size_and_align_of(fx, layout, meta); + ret.write_cvalue(fx, CValue::by_val(size, usize_layout)); + } + sym::min_align_of_val => { + intrinsic_args!(fx, args => (ptr); intrinsic); + + let layout = fx.layout_of(generic_args.type_at(0)); + // Note: Can't use is_unsized here as truly unsized types need to take the fixed size + // branch + let meta = if let Abi::ScalarPair(_, _) = ptr.layout().abi { + Some(ptr.load_scalar_pair(fx).1) + } else { + None + }; + let (_size, align) = crate::unsize::size_and_align_of(fx, layout, meta); + ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); + } + + sym::vtable_size => { + intrinsic_args!(fx, args => (vtable); intrinsic); + let vtable = vtable.load_scalar(fx); + + let size = crate::vtable::size_of_obj(fx, vtable); + ret.write_cvalue(fx, CValue::by_val(size, usize_layout)); + } + + sym::vtable_align => { + intrinsic_args!(fx, args => (vtable); intrinsic); + let vtable = vtable.load_scalar(fx); + + let align = crate::vtable::min_align_of_obj(fx, vtable); + ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); + } + + sym::exact_div => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + // FIXME trap on inexact + let res = crate::num::codegen_int_binop(fx, BinOp::Div, x, y); + ret.write_cvalue(fx, res); + } + sym::saturating_add | sym::saturating_sub => { + intrinsic_args!(fx, args => (lhs, rhs); intrinsic); + + assert_eq!(lhs.layout().ty, rhs.layout().ty); + let bin_op = match intrinsic { + sym::saturating_add => BinOp::Add, + sym::saturating_sub => BinOp::Sub, + _ => unreachable!(), + }; + + let res = crate::num::codegen_saturating_int_binop(fx, bin_op, lhs, rhs); + ret.write_cvalue(fx, res); + } + sym::rotate_left => { + intrinsic_args!(fx, args => (x, y); intrinsic); + let y = y.load_scalar(fx); + + let layout = x.layout(); + let x = x.load_scalar(fx); + let res = fx.bcx.ins().rotl(x, y); + ret.write_cvalue(fx, CValue::by_val(res, layout)); + } + sym::rotate_right => { + intrinsic_args!(fx, args => (x, y); intrinsic); + let y = y.load_scalar(fx); + + let layout = x.layout(); + let x = x.load_scalar(fx); + let res = fx.bcx.ins().rotr(x, y); + ret.write_cvalue(fx, CValue::by_val(res, layout)); + } + + // The only difference between offset and arith_offset is regarding UB. Because Cranelift + // doesn't have UB both are codegen'ed the same way + sym::arith_offset => { + intrinsic_args!(fx, args => (base, offset); intrinsic); + let offset = offset.load_scalar(fx); + + let pointee_ty = base.layout().ty.builtin_deref(true).unwrap(); + let pointee_size = fx.layout_of(pointee_ty).size.bytes(); + let ptr_diff = if pointee_size != 1 { + fx.bcx.ins().imul_imm(offset, pointee_size as i64) + } else { + offset + }; + let base_val = base.load_scalar(fx); + let res = fx.bcx.ins().iadd(base_val, ptr_diff); + ret.write_cvalue(fx, CValue::by_val(res, base.layout())); + } + + sym::ptr_mask => { + intrinsic_args!(fx, args => (ptr, mask); intrinsic); + let ptr = ptr.load_scalar(fx); + let mask = mask.load_scalar(fx); + fx.bcx.ins().band(ptr, mask); + } + + sym::write_bytes | sym::volatile_set_memory => { + intrinsic_args!(fx, args => (dst, val, count); intrinsic); + let val = val.load_scalar(fx); + let count = count.load_scalar(fx); + + let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap(); + let pointee_size = fx.layout_of(pointee_ty).size.bytes(); + let count = if pointee_size != 1 { + fx.bcx.ins().imul_imm(count, pointee_size as i64) + } else { + count + }; + let dst_ptr = dst.load_scalar(fx); + // FIXME make the memset actually volatile when switching to emit_small_memset + // FIXME use emit_small_memset + fx.bcx.call_memset(fx.target_config, dst_ptr, val, count); + } + sym::ctlz | sym::ctlz_nonzero => { + intrinsic_args!(fx, args => (arg); intrinsic); + let val = arg.load_scalar(fx); + + // FIXME trap on `ctlz_nonzero` with zero arg. + let res = fx.bcx.ins().clz(val); + let res = clif_intcast(fx, res, types::I32, false); + let res = CValue::by_val(res, ret.layout()); + ret.write_cvalue(fx, res); + } + sym::cttz | sym::cttz_nonzero => { + intrinsic_args!(fx, args => (arg); intrinsic); + let val = arg.load_scalar(fx); + + // FIXME trap on `cttz_nonzero` with zero arg. + let res = fx.bcx.ins().ctz(val); + let res = clif_intcast(fx, res, types::I32, false); + let res = CValue::by_val(res, ret.layout()); + ret.write_cvalue(fx, res); + } + sym::ctpop => { + intrinsic_args!(fx, args => (arg); intrinsic); + let val = arg.load_scalar(fx); + + let res = fx.bcx.ins().popcnt(val); + let res = clif_intcast(fx, res, types::I32, false); + let res = CValue::by_val(res, ret.layout()); + ret.write_cvalue(fx, res); + } + sym::bitreverse => { + intrinsic_args!(fx, args => (arg); intrinsic); + let val = arg.load_scalar(fx); + + let res = fx.bcx.ins().bitrev(val); + let res = CValue::by_val(res, arg.layout()); + ret.write_cvalue(fx, res); + } + sym::bswap => { + intrinsic_args!(fx, args => (arg); intrinsic); + let val = arg.load_scalar(fx); + + let res = if fx.bcx.func.dfg.value_type(val) == types::I8 { + val + } else { + fx.bcx.ins().bswap(val) + }; + let res = CValue::by_val(res, arg.layout()); + ret.write_cvalue(fx, res); + } + sym::assert_inhabited | sym::assert_zero_valid | sym::assert_mem_uninitialized_valid => { + intrinsic_args!(fx, args => (); intrinsic); + + let ty = generic_args.type_at(0); + + let requirement = ValidityRequirement::from_intrinsic(intrinsic); + + if let Some(requirement) = requirement { + let do_panic = !fx + .tcx + .check_validity_requirement((requirement, fx.param_env().and(ty))) + .expect("expect to have layout during codegen"); + + if do_panic { + let layout = fx.layout_of(ty); + let msg_str = with_no_visible_paths!({ + with_no_trimmed_paths!({ + if layout.abi.is_uninhabited() { + // Use this error even for the other intrinsics as it is more precise. + format!("attempted to instantiate uninhabited type `{}`", ty) + } else if intrinsic == sym::assert_zero_valid { + format!( + "attempted to zero-initialize type `{}`, which is invalid", + ty + ) + } else { + format!( + "attempted to leave type `{}` uninitialized, which is invalid", + ty + ) + } + }) + }); + crate::base::codegen_panic_nounwind(fx, &msg_str, Some(source_info.span)); + return Ok(()); + } + } + } + + sym::volatile_load | sym::unaligned_volatile_load => { + intrinsic_args!(fx, args => (ptr); intrinsic); + + // Cranelift treats loads as volatile by default + // FIXME correctly handle unaligned_volatile_load + let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap()); + let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout); + ret.write_cvalue(fx, val); + } + sym::volatile_store | sym::unaligned_volatile_store | sym::nontemporal_store => { + intrinsic_args!(fx, args => (ptr, val); intrinsic); + let ptr = ptr.load_scalar(fx); + + // Cranelift treats stores as volatile by default + // FIXME correctly handle unaligned_volatile_store + // FIXME actually do nontemporal stores if requested + let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); + dest.write_cvalue(fx, val); + } + + sym::pref_align_of + | sym::needs_drop + | sym::type_id + | sym::type_name + | sym::variant_count => { + intrinsic_args!(fx, args => (); intrinsic); + + let const_val = fx + .tcx + .const_eval_instance(ParamEnv::reveal_all(), instance, source_info.span) + .unwrap(); + let val = crate::constant::codegen_const_value(fx, const_val, ret.layout().ty); + ret.write_cvalue(fx, val); + } + + sym::ptr_offset_from | sym::ptr_offset_from_unsigned => { + intrinsic_args!(fx, args => (ptr, base); intrinsic); + let ptr = ptr.load_scalar(fx); + let base = base.load_scalar(fx); + let ty = generic_args.type_at(0); + + let pointee_size: u64 = fx.layout_of(ty).size.bytes(); + let diff_bytes = fx.bcx.ins().isub(ptr, base); + // FIXME this can be an exact division. + let val = if intrinsic == sym::ptr_offset_from_unsigned { + let usize_layout = fx.layout_of(fx.tcx.types.usize); + // Because diff_bytes ULE isize::MAX, this would be fine as signed, + // but unsigned is slightly easier to codegen, so might as well. + CValue::by_val(fx.bcx.ins().udiv_imm(diff_bytes, pointee_size as i64), usize_layout) + } else { + let isize_layout = fx.layout_of(fx.tcx.types.isize); + CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout) + }; + ret.write_cvalue(fx, val); + } + + sym::caller_location => { + intrinsic_args!(fx, args => (); intrinsic); + + let caller_location = fx.get_caller_location(source_info); + ret.write_cvalue(fx, caller_location); + } + + _ if intrinsic.as_str().starts_with("atomic_fence") => { + intrinsic_args!(fx, args => (); intrinsic); + + fx.bcx.ins().fence(); + } + _ if intrinsic.as_str().starts_with("atomic_singlethreadfence") => { + intrinsic_args!(fx, args => (); intrinsic); + + // FIXME use a compiler fence once Cranelift supports it + fx.bcx.ins().fence(); + } + _ if intrinsic.as_str().starts_with("atomic_load") => { + intrinsic_args!(fx, args => (ptr); intrinsic); + let ptr = ptr.load_scalar(fx); + + let ty = generic_args.type_at(0); + match ty.kind() { + ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { + // FIXME implement 128bit atomics + if fx.tcx.is_compiler_builtins(LOCAL_CRATE) { + // special case for compiler-builtins to avoid having to patch it + crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported"); + return Ok(()); + } else { + fx.tcx + .dcx() + .span_fatal(source_info.span, "128bit atomics not yet supported"); + } + } + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty); + return Ok(()); + } + } + let clif_ty = fx.clif_type(ty).unwrap(); + + let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr); + + let val = CValue::by_val(val, fx.layout_of(ty)); + ret.write_cvalue(fx, val); + } + _ if intrinsic.as_str().starts_with("atomic_store") => { + intrinsic_args!(fx, args => (ptr, val); intrinsic); + let ptr = ptr.load_scalar(fx); + + let ty = generic_args.type_at(0); + match ty.kind() { + ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { + // FIXME implement 128bit atomics + if fx.tcx.is_compiler_builtins(LOCAL_CRATE) { + // special case for compiler-builtins to avoid having to patch it + crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported"); + return Ok(()); + } else { + fx.tcx + .dcx() + .span_fatal(source_info.span, "128bit atomics not yet supported"); + } + } + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty); + return Ok(()); + } + } + + let val = val.load_scalar(fx); + + fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr); + } + _ if intrinsic.as_str().starts_with("atomic_xchg") => { + intrinsic_args!(fx, args => (ptr, new); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = new.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let new = new.load_scalar(fx); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_cxchg") => { + // both atomic_cxchg_* and atomic_cxchgweak_* + intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = new.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + + let test_old = test_old.load_scalar(fx); + let new = new.load_scalar(fx); + + let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new); + let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old); + + let ret_val = CValue::by_val_pair(old, is_eq, ret.layout()); + ret.write_cvalue(fx, ret_val) + } + + _ if intrinsic.as_str().starts_with("atomic_xadd") => { + intrinsic_args!(fx, args => (ptr, amount); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = amount.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let amount = amount.load_scalar(fx); + + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_xsub") => { + intrinsic_args!(fx, args => (ptr, amount); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = amount.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let amount = amount.load_scalar(fx); + + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_and") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = src.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let src = src.load_scalar(fx); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_or") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = src.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let src = src.load_scalar(fx); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_xor") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = src.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let src = src.load_scalar(fx); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_nand") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = src.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let src = src.load_scalar(fx); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_max") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = src.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let src = src.load_scalar(fx); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_umax") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = src.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let src = src.load_scalar(fx); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_min") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = src.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let src = src.load_scalar(fx); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + _ if intrinsic.as_str().starts_with("atomic_umin") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + + let layout = src.layout(); + match layout.ty.kind() { + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + _ => { + report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); + return Ok(()); + } + } + let ty = fx.clif_type(layout.ty).unwrap(); + + let src = src.load_scalar(fx); + + let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src); + + let old = CValue::by_val(old, layout); + ret.write_cvalue(fx, old); + } + + sym::minnumf32 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let val = crate::num::codegen_float_min(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32)); + ret.write_cvalue(fx, val); + } + sym::minnumf64 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let val = crate::num::codegen_float_min(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); + ret.write_cvalue(fx, val); + } + sym::maxnumf32 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let val = crate::num::codegen_float_max(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32)); + ret.write_cvalue(fx, val); + } + sym::maxnumf64 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let val = crate::num::codegen_float_max(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); + ret.write_cvalue(fx, val); + } + + sym::catch_unwind => { + intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic); + let f = f.load_scalar(fx); + let data = data.load_scalar(fx); + let _catch_fn = catch_fn.load_scalar(fx); + + // FIXME once unwinding is supported, change this to actually catch panics + let f_sig = fx.bcx.func.import_signature(Signature { + call_conv: fx.target_config.default_call_conv, + params: vec![AbiParam::new(pointer_ty(fx.tcx))], + returns: vec![], + }); + + fx.bcx.ins().call_indirect(f_sig, f, &[data]); + + let layout = fx.layout_of(fx.tcx.types.i32); + let ret_val = CValue::by_val(fx.bcx.ins().iconst(types::I32, 0), layout); + ret.write_cvalue(fx, ret_val); + } + + sym::fadd_fast + | sym::fsub_fast + | sym::fmul_fast + | sym::fdiv_fast + | sym::frem_fast + | sym::fadd_algebraic + | sym::fsub_algebraic + | sym::fmul_algebraic + | sym::fdiv_algebraic + | sym::frem_algebraic => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + let res = crate::num::codegen_float_binop( + fx, + match intrinsic { + sym::fadd_fast | sym::fadd_algebraic => BinOp::Add, + sym::fsub_fast | sym::fsub_algebraic => BinOp::Sub, + sym::fmul_fast | sym::fmul_algebraic => BinOp::Mul, + sym::fdiv_fast | sym::fdiv_algebraic => BinOp::Div, + sym::frem_fast | sym::frem_algebraic => BinOp::Rem, + _ => unreachable!(), + }, + x, + y, + ); + ret.write_cvalue(fx, res); + } + sym::float_to_int_unchecked => { + intrinsic_args!(fx, args => (f); intrinsic); + let f = f.load_scalar(fx); + + let res = crate::cast::clif_int_or_float_cast( + fx, + f, + false, + fx.clif_type(ret.layout().ty).unwrap(), + type_sign(ret.layout().ty), + ); + ret.write_cvalue(fx, CValue::by_val(res, ret.layout())); + } + + sym::raw_eq => { + intrinsic_args!(fx, args => (lhs_ref, rhs_ref); intrinsic); + let lhs_ref = lhs_ref.load_scalar(fx); + let rhs_ref = rhs_ref.load_scalar(fx); + + let size = fx.layout_of(generic_args.type_at(0)).layout.size(); + // FIXME add and use emit_small_memcmp + let is_eq_value = if size == Size::ZERO { + // No bytes means they're trivially equal + fx.bcx.ins().iconst(types::I8, 1) + } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) { + // Can't use `trusted` for these loads; they could be unaligned. + let mut flags = MemFlags::new(); + flags.set_notrap(); + let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0); + let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0); + fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val) + } else { + // Just call `memcmp` (like slices do in core) when the + // size is too large or it's not a power-of-two. + let signed_bytes = i64::try_from(size.bytes()).unwrap(); + let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes); + let params = vec![AbiParam::new(fx.pointer_type); 3]; + let returns = vec![AbiParam::new(types::I32)]; + let args = &[lhs_ref, rhs_ref, bytes_val]; + let cmp = fx.lib_call("memcmp", params, returns, args)[0]; + fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0) + }; + ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout())); + } + + sym::compare_bytes => { + intrinsic_args!(fx, args => (lhs_ptr, rhs_ptr, bytes_val); intrinsic); + let lhs_ptr = lhs_ptr.load_scalar(fx); + let rhs_ptr = rhs_ptr.load_scalar(fx); + let bytes_val = bytes_val.load_scalar(fx); + + let params = vec![AbiParam::new(fx.pointer_type); 3]; + let returns = vec![AbiParam::new(types::I32)]; + let args = &[lhs_ptr, rhs_ptr, bytes_val]; + // Here we assume that the `memcmp` provided by the target is a NOP for size 0. + let cmp = fx.lib_call("memcmp", params, returns, args)[0]; + ret.write_cvalue(fx, CValue::by_val(cmp, ret.layout())); + } + + sym::black_box => { + intrinsic_args!(fx, args => (a); intrinsic); + + // FIXME implement black_box semantics + ret.write_cvalue(fx, a); + } + + // FIXME implement variadics in cranelift + sym::va_copy | sym::va_arg | sym::va_end => { + fx.tcx.dcx().span_fatal( + source_info.span, + "Defining variadic functions is not yet supported by Cranelift", + ); + } + + // Unimplemented intrinsics must have a fallback body. The fallback body is obtained + // by converting the `InstanceKind::Intrinsic` to an `InstanceKind::Item`. + _ => { + let intrinsic = fx.tcx.intrinsic(instance.def_id()).unwrap(); + if intrinsic.must_be_overridden { + span_bug!( + source_info.span, + "intrinsic {} must be overridden by codegen_cranelift, but isn't", + intrinsic.name, + ); + } + return Err(Instance::new(instance.def_id(), instance.args)); + } + } + + let ret_block = fx.get_block(destination.unwrap()); + fx.bcx.ins().jump(ret_block, &[]); + Ok(()) +} diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs new file mode 100644 index 00000000000..ca910dccb0d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -0,0 +1,1160 @@ +//! Codegen `extern "platform-intrinsic"` intrinsics. + +use cranelift_codegen::ir::immediates::Offset32; +use rustc_target::abi::Endian; + +use super::*; +use crate::prelude::*; + +fn report_simd_type_validation_error( + fx: &mut FunctionCx<'_, '_, '_>, + intrinsic: Symbol, + span: Span, + ty: Ty<'_>, +) { + fx.tcx.dcx().span_err(span, format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty)); + // Prevent verifier error + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); +} + +pub(super) fn codegen_simd_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + intrinsic: Symbol, + generic_args: GenericArgsRef<'tcx>, + args: &[Spanned<mir::Operand<'tcx>>], + ret: CPlace<'tcx>, + target: BasicBlock, + span: Span, +) { + match intrinsic { + sym::simd_as | sym::simd_cast => { + intrinsic_args!(fx, args => (a); intrinsic); + + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + + simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| { + let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap(); + + let from_signed = type_sign(lane_ty); + let to_signed = type_sign(ret_lane_ty); + + clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed) + }); + } + + sym::simd_eq | sym::simd_ne | sym::simd_lt | sym::simd_le | sym::simd_gt | sym::simd_ge => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + if !x.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty); + return; + } + + // FIXME use vector instructions when possible + simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| { + let res_lane = match (lane_ty.kind(), intrinsic) { + (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane), + (ty::Uint(_), sym::simd_ne) => { + fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane) + } + (ty::Uint(_), sym::simd_lt) => { + fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane) + } + (ty::Uint(_), sym::simd_le) => { + fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane) + } + (ty::Uint(_), sym::simd_gt) => { + fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane) + } + (ty::Uint(_), sym::simd_ge) => { + fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane) + } + + (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane), + (ty::Int(_), sym::simd_ne) => { + fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane) + } + (ty::Int(_), sym::simd_lt) => { + fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane) + } + (ty::Int(_), sym::simd_le) => { + fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane) + } + (ty::Int(_), sym::simd_gt) => { + fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane) + } + (ty::Int(_), sym::simd_ge) => { + fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane) + } + + (ty::Float(_), sym::simd_eq) => { + fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane) + } + (ty::Float(_), sym::simd_ne) => { + fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane) + } + (ty::Float(_), sym::simd_lt) => { + fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane) + } + (ty::Float(_), sym::simd_le) => { + fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane) + } + (ty::Float(_), sym::simd_gt) => { + fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane) + } + (ty::Float(_), sym::simd_ge) => { + fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane) + } + + _ => unreachable!(), + }; + + bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane) + }); + } + + // simd_shuffle_generic<T, U, const I: &[u32]>(x: T, y: T) -> U + sym::simd_shuffle_generic => { + let [x, y] = args else { + bug!("wrong number of args for intrinsic {intrinsic}"); + }; + let x = codegen_operand(fx, &x.node); + let y = codegen_operand(fx, &y.node); + + if !x.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty); + return; + } + + let idx = generic_args[2] + .expect_const() + .eval(fx.tcx, ty::ParamEnv::reveal_all(), span) + .unwrap() + .1 + .unwrap_branch(); + + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + + assert_eq!(lane_ty, ret_lane_ty); + assert_eq!(idx.len() as u64, ret_lane_count); + + let total_len = lane_count * 2; + + let indexes = idx.iter().map(|idx| idx.unwrap_leaf().to_u32()).collect::<Vec<u32>>(); + + for &idx in &indexes { + assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len); + } + + for (out_idx, in_idx) in indexes.into_iter().enumerate() { + let in_lane = if u64::from(in_idx) < lane_count { + x.value_lane(fx, in_idx.into()) + } else { + y.value_lane(fx, u64::from(in_idx) - lane_count) + }; + let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap()); + out_lane.write_cvalue(fx, in_lane); + } + } + + // simd_shuffle<T, I, U>(x: T, y: T, idx: I) -> U + sym::simd_shuffle => { + let (x, y, idx) = match args { + [x, y, idx] => (x, y, idx), + _ => { + bug!("wrong number of args for intrinsic {intrinsic}"); + } + }; + let x = codegen_operand(fx, &x.node); + let y = codegen_operand(fx, &y.node); + + if !x.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty); + return; + } + + // Make sure this is actually an array, since typeck only checks the length-suffixed + // version of this intrinsic. + let idx_ty = fx.monomorphize(idx.node.ty(fx.mir, fx.tcx)); + let n: u16 = match idx_ty.kind() { + ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => len + .try_eval_target_usize(fx.tcx, ty::ParamEnv::reveal_all()) + .unwrap_or_else(|| { + span_bug!(span, "could not evaluate shuffle index array length") + }) + .try_into() + .unwrap(), + _ => { + fx.tcx.dcx().span_err( + span, + format!("simd_shuffle index must be an array of `u32`, got `{}`", idx_ty), + ); + // Prevent verifier error + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + return; + } + }; + + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + + assert_eq!(lane_ty, ret_lane_ty); + assert_eq!(u64::from(n), ret_lane_count); + + let total_len = lane_count * 2; + + let indexes = { + use rustc_middle::mir::interpret::*; + let idx_const = match &idx.node { + Operand::Constant(const_) => crate::constant::eval_mir_constant(fx, const_).0, + Operand::Copy(_) | Operand::Move(_) => unreachable!("{idx:?}"), + }; + + let idx_bytes = match idx_const { + ConstValue::Indirect { alloc_id, offset } => { + let alloc = fx.tcx.global_alloc(alloc_id).unwrap_memory(); + let size = Size::from_bytes( + 4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */ + ); + alloc + .inner() + .get_bytes_strip_provenance(fx, alloc_range(offset, size)) + .unwrap() + } + _ => unreachable!("{:?}", idx_const), + }; + + (0..ret_lane_count) + .map(|i| { + let i = usize::try_from(i).unwrap(); + let idx = rustc_middle::mir::interpret::read_target_uint( + fx.tcx.data_layout.endian, + &idx_bytes[4 * i..4 * i + 4], + ) + .expect("read_target_uint"); + u16::try_from(idx).expect("try_from u32") + }) + .collect::<Vec<u16>>() + }; + + for &idx in &indexes { + assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len); + } + + for (out_idx, in_idx) in indexes.into_iter().enumerate() { + let in_lane = if u64::from(in_idx) < lane_count { + x.value_lane(fx, in_idx.into()) + } else { + y.value_lane(fx, u64::from(in_idx) - lane_count) + }; + let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap()); + out_lane.write_cvalue(fx, in_lane); + } + } + + sym::simd_insert => { + let (base, idx, val) = match args { + [base, idx, val] => (base, idx, val), + _ => { + bug!("wrong number of args for intrinsic {intrinsic}"); + } + }; + let base = codegen_operand(fx, &base.node); + let val = codegen_operand(fx, &val.node); + + // FIXME validate + let idx_const = if let Some(idx_const) = + crate::constant::mir_operand_get_const_val(fx, &idx.node) + { + idx_const + } else { + fx.tcx.dcx().span_fatal(span, "Index argument for `simd_insert` is not a constant"); + }; + + let idx: u32 = idx_const.to_u32(); + let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx); + if u64::from(idx) >= lane_count { + fx.tcx.dcx().span_fatal( + fx.mir.span, + format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count), + ); + } + + ret.write_cvalue(fx, base); + let ret_lane = ret.place_lane(fx, idx.into()); + ret_lane.write_cvalue(fx, val); + } + + sym::simd_extract => { + let (v, idx) = match args { + [v, idx] => (v, idx), + _ => { + bug!("wrong number of args for intrinsic {intrinsic}"); + } + }; + let v = codegen_operand(fx, &v.node); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + let idx_const = if let Some(idx_const) = + crate::constant::mir_operand_get_const_val(fx, &idx.node) + { + idx_const + } else { + fx.tcx.dcx().span_warn(span, "Index argument for `simd_extract` is not a constant"); + let trap_block = fx.bcx.create_block(); + let true_ = fx.bcx.ins().iconst(types::I8, 1); + let ret_block = fx.get_block(target); + fx.bcx.ins().brif(true_, trap_block, &[], ret_block, &[]); + fx.bcx.switch_to_block(trap_block); + crate::trap::trap_unimplemented( + fx, + "Index argument for `simd_extract` is not a constant", + ); + return; + }; + + let idx = idx_const.to_u32(); + let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx); + if u64::from(idx) >= lane_count { + fx.tcx.dcx().span_fatal( + fx.mir.span, + format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count), + ); + } + + let ret_lane = v.value_lane(fx, idx.into()); + ret.write_cvalue(fx, ret_lane); + } + + sym::simd_neg + | sym::simd_bswap + | sym::simd_bitreverse + | sym::simd_ctlz + | sym::simd_ctpop + | sym::simd_cttz => { + intrinsic_args!(fx, args => (a); intrinsic); + + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + + simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| match ( + lane_ty.kind(), + intrinsic, + ) { + (ty::Int(_), sym::simd_neg) => fx.bcx.ins().ineg(lane), + (ty::Float(_), sym::simd_neg) => fx.bcx.ins().fneg(lane), + + (ty::Uint(ty::UintTy::U8) | ty::Int(ty::IntTy::I8), sym::simd_bswap) => lane, + (ty::Uint(_) | ty::Int(_), sym::simd_bswap) => fx.bcx.ins().bswap(lane), + (ty::Uint(_) | ty::Int(_), sym::simd_bitreverse) => fx.bcx.ins().bitrev(lane), + (ty::Uint(_) | ty::Int(_), sym::simd_ctlz) => fx.bcx.ins().clz(lane), + (ty::Uint(_) | ty::Int(_), sym::simd_ctpop) => fx.bcx.ins().popcnt(lane), + (ty::Uint(_) | ty::Int(_), sym::simd_cttz) => fx.bcx.ins().ctz(lane), + + _ => unreachable!(), + }); + } + + sym::simd_add + | sym::simd_sub + | sym::simd_mul + | sym::simd_div + | sym::simd_rem + | sym::simd_shl + | sym::simd_shr + | sym::simd_and + | sym::simd_or + | sym::simd_xor => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + // FIXME use vector instructions when possible + simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| { + match (lane_ty.kind(), intrinsic) { + (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane), + (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane), + (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane), + (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane), + (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane), + + (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane), + (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane), + (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane), + (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane), + (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane), + + (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane), + (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane), + (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane), + (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane), + (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call( + "fmodf", + vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], + vec![AbiParam::new(types::F32)], + &[x_lane, y_lane], + )[0], + (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call( + "fmod", + vec![AbiParam::new(types::F64), AbiParam::new(types::F64)], + vec![AbiParam::new(types::F64)], + &[x_lane, y_lane], + )[0], + + (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane), + (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane), + (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane), + (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane), + (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane), + + (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane), + (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane), + (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane), + (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane), + (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane), + + _ => unreachable!(), + } + }); + } + + sym::simd_fma => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + assert_eq!(a.layout(), b.layout()); + assert_eq!(a.layout(), c.layout()); + assert_eq!(a.layout(), ret.layout()); + + let layout = a.layout(); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let res_lane_layout = fx.layout_of(lane_ty); + + for lane in 0..lane_count { + let a_lane = a.value_lane(fx, lane).load_scalar(fx); + let b_lane = b.value_lane(fx, lane).load_scalar(fx); + let c_lane = c.value_lane(fx, lane).load_scalar(fx); + + let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane); + let res_lane = CValue::by_val(res_lane, res_lane_layout); + + ret.place_lane(fx, lane).write_cvalue(fx, res_lane); + } + } + + sym::simd_fmin | sym::simd_fmax => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + if !x.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty); + return; + } + + // FIXME use vector instructions when possible + simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| { + match lane_ty.kind() { + ty::Float(_) => {} + _ => unreachable!("{:?}", lane_ty), + } + match intrinsic { + sym::simd_fmin => crate::num::codegen_float_min(fx, x_lane, y_lane), + sym::simd_fmax => crate::num::codegen_float_max(fx, x_lane, y_lane), + _ => unreachable!(), + } + }); + } + + sym::simd_fpow => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + + simd_pair_for_each_lane(fx, a, b, ret, &|fx, lane_ty, _ret_lane_ty, a_lane, b_lane| { + match lane_ty.kind() { + ty::Float(FloatTy::F32) => fx.lib_call( + "powf", + vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], + vec![AbiParam::new(types::F32)], + &[a_lane, b_lane], + )[0], + ty::Float(FloatTy::F64) => fx.lib_call( + "pow", + vec![AbiParam::new(types::F64), AbiParam::new(types::F64)], + vec![AbiParam::new(types::F64)], + &[a_lane, b_lane], + )[0], + _ => unreachable!("{:?}", lane_ty), + } + }); + } + + sym::simd_fpowi => { + intrinsic_args!(fx, args => (a, exp); intrinsic); + let exp = exp.load_scalar(fx); + + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + + simd_for_each_lane( + fx, + a, + ret, + &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() { + ty::Float(FloatTy::F32) => fx.lib_call( + "__powisf2", // compiler-builtins + vec![AbiParam::new(types::F32), AbiParam::new(types::I32)], + vec![AbiParam::new(types::F32)], + &[lane, exp], + )[0], + ty::Float(FloatTy::F64) => fx.lib_call( + "__powidf2", // compiler-builtins + vec![AbiParam::new(types::F64), AbiParam::new(types::I32)], + vec![AbiParam::new(types::F64)], + &[lane, exp], + )[0], + _ => unreachable!("{:?}", lane_ty), + }, + ); + } + + sym::simd_fsin + | sym::simd_fcos + | sym::simd_fexp + | sym::simd_fexp2 + | sym::simd_flog + | sym::simd_flog10 + | sym::simd_flog2 + | sym::simd_round => { + intrinsic_args!(fx, args => (a); intrinsic); + + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + + simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| { + let lane_ty = match lane_ty.kind() { + ty::Float(FloatTy::F32) => types::F32, + ty::Float(FloatTy::F64) => types::F64, + _ => unreachable!("{:?}", lane_ty), + }; + let name = match (intrinsic, lane_ty) { + (sym::simd_fsin, types::F32) => "sinf", + (sym::simd_fsin, types::F64) => "sin", + (sym::simd_fcos, types::F32) => "cosf", + (sym::simd_fcos, types::F64) => "cos", + (sym::simd_fexp, types::F32) => "expf", + (sym::simd_fexp, types::F64) => "exp", + (sym::simd_fexp2, types::F32) => "exp2f", + (sym::simd_fexp2, types::F64) => "exp2", + (sym::simd_flog, types::F32) => "logf", + (sym::simd_flog, types::F64) => "log", + (sym::simd_flog10, types::F32) => "log10f", + (sym::simd_flog10, types::F64) => "log10", + (sym::simd_flog2, types::F32) => "log2f", + (sym::simd_flog2, types::F64) => "log2", + (sym::simd_round, types::F32) => "roundf", + (sym::simd_round, types::F64) => "round", + _ => unreachable!("{:?}", intrinsic), + }; + fx.lib_call( + name, + vec![AbiParam::new(lane_ty)], + vec![AbiParam::new(lane_ty)], + &[lane], + )[0] + }); + } + + sym::simd_fabs | sym::simd_fsqrt | sym::simd_ceil | sym::simd_floor | sym::simd_trunc => { + intrinsic_args!(fx, args => (a); intrinsic); + + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + + simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| { + match lane_ty.kind() { + ty::Float(_) => {} + _ => unreachable!("{:?}", lane_ty), + } + match intrinsic { + sym::simd_fabs => fx.bcx.ins().fabs(lane), + sym::simd_fsqrt => fx.bcx.ins().sqrt(lane), + sym::simd_ceil => fx.bcx.ins().ceil(lane), + sym::simd_floor => fx.bcx.ins().floor(lane), + sym::simd_trunc => fx.bcx.ins().trunc(lane), + _ => unreachable!(), + } + }); + } + + sym::simd_reduce_add_ordered => { + intrinsic_args!(fx, args => (v, acc); intrinsic); + let acc = acc.load_scalar(fx); + + // FIXME there must be no acc param for integer vectors + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| { + if lane_ty.is_floating_point() { + fx.bcx.ins().fadd(a, b) + } else { + fx.bcx.ins().iadd(a, b) + } + }); + } + + sym::simd_reduce_add_unordered => { + intrinsic_args!(fx, args => (v); intrinsic); + + // FIXME there must be no acc param for integer vectors + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce(fx, v, None, ret, &|fx, lane_ty, a, b| { + if lane_ty.is_floating_point() { + fx.bcx.ins().fadd(a, b) + } else { + fx.bcx.ins().iadd(a, b) + } + }); + } + + sym::simd_reduce_mul_ordered => { + intrinsic_args!(fx, args => (v, acc); intrinsic); + let acc = acc.load_scalar(fx); + + // FIXME there must be no acc param for integer vectors + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| { + if lane_ty.is_floating_point() { + fx.bcx.ins().fmul(a, b) + } else { + fx.bcx.ins().imul(a, b) + } + }); + } + + sym::simd_reduce_mul_unordered => { + intrinsic_args!(fx, args => (v); intrinsic); + + // FIXME there must be no acc param for integer vectors + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce(fx, v, None, ret, &|fx, lane_ty, a, b| { + if lane_ty.is_floating_point() { + fx.bcx.ins().fmul(a, b) + } else { + fx.bcx.ins().imul(a, b) + } + }); + } + + sym::simd_reduce_all => { + intrinsic_args!(fx, args => (v); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b)); + } + + sym::simd_reduce_any => { + intrinsic_args!(fx, args => (v); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b)); + } + + sym::simd_reduce_and => { + intrinsic_args!(fx, args => (v); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b)); + } + + sym::simd_reduce_or => { + intrinsic_args!(fx, args => (v); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b)); + } + + sym::simd_reduce_xor => { + intrinsic_args!(fx, args => (v); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b)); + } + + sym::simd_reduce_min => { + intrinsic_args!(fx, args => (v); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce(fx, v, None, ret, &|fx, ty, a, b| { + let lt = match ty.kind() { + ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b), + ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b), + ty::Float(_) => return crate::num::codegen_float_min(fx, a, b), + _ => unreachable!(), + }; + fx.bcx.ins().select(lt, a, b) + }); + } + + sym::simd_reduce_max => { + intrinsic_args!(fx, args => (v); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + simd_reduce(fx, v, None, ret, &|fx, ty, a, b| { + let gt = match ty.kind() { + ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b), + ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b), + ty::Float(_) => return crate::num::codegen_float_max(fx, a, b), + _ => unreachable!(), + }; + fx.bcx.ins().select(gt, a, b) + }); + } + + sym::simd_select => { + intrinsic_args!(fx, args => (m, a, b); intrinsic); + + if !m.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty); + return; + } + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + assert_eq!(a.layout(), b.layout()); + + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + + for lane in 0..lane_count { + let m_lane = m.value_lane(fx, lane).load_scalar(fx); + let a_lane = a.value_lane(fx, lane).load_scalar(fx); + let b_lane = b.value_lane(fx, lane).load_scalar(fx); + + let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0); + let res_lane = + CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout); + + ret.place_lane(fx, lane).write_cvalue(fx, res_lane); + } + } + + sym::simd_select_bitmask => { + intrinsic_args!(fx, args => (m, a, b); intrinsic); + + if !a.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); + return; + } + assert_eq!(a.layout(), b.layout()); + + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + + let expected_int_bits = lane_count.max(8); + let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); + + let m = match m.layout().ty.kind() { + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => m.load_scalar(fx), + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_target_usize(fx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + m.force_stack(fx).0.load( + fx, + Type::int(expected_int_bits as u16).unwrap(), + MemFlags::trusted(), + ) + } + _ => { + fx.tcx.dcx().span_fatal( + span, + format!( + "invalid monomorphization of `simd_select_bitmask` intrinsic: \ + cannot accept `{}` as mask, expected `u{}` or `[u8; {}]`", + ret.layout().ty, + expected_int_bits, + expected_bytes + ), + ); + } + }; + + for lane in 0..lane_count { + // The bit order of the mask depends on the byte endianness, LSB-first for + // little endian and MSB-first for big endian. + let mask_lane = match fx.tcx.sess.target.endian { + Endian::Big => lane_count - 1 - lane, + Endian::Little => lane, + }; + let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(mask_lane) as i64); + let m_lane = fx.bcx.ins().band_imm(m_lane, 1); + let a_lane = a.value_lane(fx, lane).load_scalar(fx); + let b_lane = b.value_lane(fx, lane).load_scalar(fx); + + let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0); + let res_lane = + CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout); + + ret.place_lane(fx, lane).write_cvalue(fx, res_lane); + } + } + + sym::simd_bitmask => { + intrinsic_args!(fx, args => (a); intrinsic); + + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); + let lane_clif_ty = fx.clif_type(lane_ty).unwrap(); + + // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a + // vector mask and returns the most significant bit (MSB) of each lane in the form + // of either: + // * an unsigned integer + // * an array of `u8` + // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. + // + // The bit order of the result depends on the byte endianness, LSB-first for little + // endian and MSB-first for big endian. + let expected_int_bits = lane_count.max(8); + let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); + + match lane_ty.kind() { + ty::Int(_) | ty::Uint(_) => {} + _ => { + fx.tcx.dcx().span_fatal( + span, + format!( + "invalid monomorphization of `simd_bitmask` intrinsic: \ + vector argument `{}`'s element type `{}`, expected integer element \ + type", + a.layout().ty, + lane_ty + ), + ); + } + } + + let res_type = + Type::int_with_byte_size(u16::try_from(expected_bytes).unwrap()).unwrap(); + let mut res = type_zero_value(&mut fx.bcx, res_type); + + let lanes = match fx.tcx.sess.target.endian { + Endian::Big => Box::new(0..lane_count) as Box<dyn Iterator<Item = u64>>, + Endian::Little => Box::new((0..lane_count).rev()) as Box<dyn Iterator<Item = u64>>, + }; + for lane in lanes { + let a_lane = a.value_lane(fx, lane).load_scalar(fx); + + // extract sign bit of an int + let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_clif_ty.bits() - 1)); + + // shift sign bit into result + let a_lane_sign = clif_intcast(fx, a_lane_sign, res_type, false); + res = fx.bcx.ins().ishl_imm(res, 1); + res = fx.bcx.ins().bor(res, a_lane_sign); + } + + match ret.layout().ty.kind() { + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {} + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_target_usize(fx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => {} + _ => { + fx.tcx.dcx().span_fatal( + span, + format!( + "invalid monomorphization of `simd_bitmask` intrinsic: \ + cannot return `{}`, expected `u{}` or `[u8; {}]`", + ret.layout().ty, + expected_int_bits, + expected_bytes + ), + ); + } + } + + let res = CValue::by_val(res, ret.layout()); + ret.write_cvalue(fx, res); + } + + sym::simd_saturating_add | sym::simd_saturating_sub => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + let bin_op = match intrinsic { + sym::simd_saturating_add => BinOp::Add, + sym::simd_saturating_sub => BinOp::Sub, + _ => unreachable!(), + }; + + // FIXME use vector instructions when possible + simd_pair_for_each_lane_typed(fx, x, y, ret, &|fx, x_lane, y_lane| { + crate::num::codegen_saturating_int_binop(fx, bin_op, x_lane, y_lane) + }); + } + + sym::simd_expose_provenance | sym::simd_with_exposed_provenance | sym::simd_cast_ptr => { + intrinsic_args!(fx, args => (arg); intrinsic); + ret.write_cvalue_transmute(fx, arg); + } + + sym::simd_arith_offset => { + intrinsic_args!(fx, args => (ptr, offset); intrinsic); + + let (lane_count, ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx); + let pointee_ty = ptr_lane_ty.builtin_deref(true).unwrap(); + let pointee_size = fx.layout_of(pointee_ty).size.bytes(); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + assert_eq!(lane_count, ret_lane_count); + + for lane_idx in 0..lane_count { + let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx); + let offset_lane = offset.value_lane(fx, lane_idx).load_scalar(fx); + + let ptr_diff = if pointee_size != 1 { + fx.bcx.ins().imul_imm(offset_lane, pointee_size as i64) + } else { + offset_lane + }; + let res_lane = fx.bcx.ins().iadd(ptr_lane, ptr_diff); + let res_lane = CValue::by_val(res_lane, ret_lane_layout); + + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } + } + + sym::simd_masked_store => { + intrinsic_args!(fx, args => (mask, ptr, val); intrinsic); + + let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(val_lane_count, mask_lane_count); + let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap(); + let ptr_val = ptr.load_scalar(fx); + + for lane_idx in 0..val_lane_count { + let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx); + let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx); + + let if_enabled = fx.bcx.create_block(); + let next = fx.bcx.create_block(); + + fx.bcx.ins().brif(mask_lane, if_enabled, &[], next, &[]); + fx.bcx.seal_block(if_enabled); + + fx.bcx.switch_to_block(if_enabled); + let offset = lane_idx as i32 * lane_clif_ty.bytes() as i32; + fx.bcx.ins().store(MemFlags::trusted(), val_lane, ptr_val, Offset32::new(offset)); + fx.bcx.ins().jump(next, &[]); + + fx.bcx.seal_block(next); + fx.bcx.switch_to_block(next); + + fx.bcx.ins().nop(); + } + } + + sym::simd_gather => { + intrinsic_args!(fx, args => (val, ptr, mask); intrinsic); + + let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx); + let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(val_lane_count, ptr_lane_count); + assert_eq!(val_lane_count, mask_lane_count); + assert_eq!(val_lane_count, ret_lane_count); + + let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap(); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + + for lane_idx in 0..ptr_lane_count { + let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx); + let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx); + let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx); + + let if_enabled = fx.bcx.create_block(); + let if_disabled = fx.bcx.create_block(); + let next = fx.bcx.create_block(); + let res_lane = fx.bcx.append_block_param(next, lane_clif_ty); + + fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]); + fx.bcx.seal_block(if_enabled); + fx.bcx.seal_block(if_disabled); + + fx.bcx.switch_to_block(if_enabled); + let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0); + fx.bcx.ins().jump(next, &[res]); + + fx.bcx.switch_to_block(if_disabled); + fx.bcx.ins().jump(next, &[val_lane]); + + fx.bcx.seal_block(next); + fx.bcx.switch_to_block(next); + + fx.bcx.ins().nop(); + + ret.place_lane(fx, lane_idx) + .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout)); + } + } + + sym::simd_masked_load => { + intrinsic_args!(fx, args => (mask, ptr, val); intrinsic); + + let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(val_lane_count, mask_lane_count); + assert_eq!(val_lane_count, ret_lane_count); + + let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap(); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + let ptr_val = ptr.load_scalar(fx); + + for lane_idx in 0..ret_lane_count { + let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx); + let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx); + + let if_enabled = fx.bcx.create_block(); + let if_disabled = fx.bcx.create_block(); + let next = fx.bcx.create_block(); + let res_lane = fx.bcx.append_block_param(next, lane_clif_ty); + + fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]); + fx.bcx.seal_block(if_enabled); + fx.bcx.seal_block(if_disabled); + + fx.bcx.switch_to_block(if_enabled); + let offset = lane_idx as i32 * lane_clif_ty.bytes() as i32; + let res = fx.bcx.ins().load( + lane_clif_ty, + MemFlags::trusted(), + ptr_val, + Offset32::new(offset), + ); + fx.bcx.ins().jump(next, &[res]); + + fx.bcx.switch_to_block(if_disabled); + fx.bcx.ins().jump(next, &[val_lane]); + + fx.bcx.seal_block(next); + fx.bcx.switch_to_block(next); + + fx.bcx.ins().nop(); + + ret.place_lane(fx, lane_idx) + .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout)); + } + } + + sym::simd_scatter => { + intrinsic_args!(fx, args => (val, ptr, mask); intrinsic); + + let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx); + let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx); + assert_eq!(val_lane_count, ptr_lane_count); + assert_eq!(val_lane_count, mask_lane_count); + + for lane_idx in 0..ptr_lane_count { + let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx); + let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx); + let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx); + + let if_enabled = fx.bcx.create_block(); + let next = fx.bcx.create_block(); + + fx.bcx.ins().brif(mask_lane, if_enabled, &[], next, &[]); + fx.bcx.seal_block(if_enabled); + + fx.bcx.switch_to_block(if_enabled); + fx.bcx.ins().store(MemFlags::trusted(), val_lane, ptr_lane, 0); + fx.bcx.ins().jump(next, &[]); + + fx.bcx.seal_block(next); + fx.bcx.switch_to_block(next); + } + } + + _ => { + fx.tcx.dcx().span_err(span, format!("Unknown SIMD intrinsic {}", intrinsic)); + // Prevent verifier error + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + return; + } + } + let ret_block = fx.get_block(target); + fx.bcx.ins().jump(ret_block, &[]); +} diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs new file mode 100644 index 00000000000..f737af25b62 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -0,0 +1,360 @@ +// tidy-alphabetical-start +#![allow(rustc::diagnostic_outside_of_impl)] +#![allow(rustc::untranslatable_diagnostic)] +#![cfg_attr(doc, allow(internal_features))] +#![cfg_attr(doc, doc(rust_logo))] +#![cfg_attr(doc, feature(rustdoc_internals))] +// Note: please avoid adding other feature gates where possible +#![feature(rustc_private)] +// Note: please avoid adding other feature gates where possible +#![warn(rust_2018_idioms)] +#![warn(unreachable_pub)] +#![warn(unused_lifetimes)] +// tidy-alphabetical-end + +extern crate jobserver; +#[macro_use] +extern crate rustc_middle; +extern crate rustc_ast; +extern crate rustc_codegen_ssa; +extern crate rustc_data_structures; +extern crate rustc_errors; +extern crate rustc_fs_util; +extern crate rustc_hir; +extern crate rustc_incremental; +extern crate rustc_index; +extern crate rustc_metadata; +extern crate rustc_session; +extern crate rustc_span; +extern crate rustc_target; + +// This prevents duplicating functions and statics that are already part of the host rustc process. +#[allow(unused_extern_crates)] +extern crate rustc_driver; + +use std::any::Any; +use std::cell::{Cell, RefCell}; +use std::sync::Arc; + +use cranelift_codegen::isa::TargetIsa; +use cranelift_codegen::settings::{self, Configurable}; +use rustc_codegen_ssa::traits::CodegenBackend; +use rustc_codegen_ssa::CodegenResults; +use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_errors::ErrorGuaranteed; +use rustc_metadata::EncodedMetadata; +use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_session::config::OutputFilenames; +use rustc_session::Session; +use rustc_span::{sym, Symbol}; + +pub use crate::config::*; +use crate::prelude::*; + +mod abi; +mod allocator; +mod analyze; +mod archive; +mod base; +mod cast; +mod codegen_i128; +mod common; +mod compiler_builtins; +mod concurrency_limiter; +mod config; +mod constant; +mod debuginfo; +mod discriminant; +mod driver; +mod global_asm; +mod inline_asm; +mod intrinsics; +mod linkage; +mod main_shim; +mod num; +mod optimize; +mod pointer; +mod pretty_clif; +mod toolchain; +mod trap; +mod unsize; +mod unwind_module; +mod value_and_place; +mod vtable; + +mod prelude { + pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; + pub(crate) use cranelift_codegen::ir::function::Function; + pub(crate) use cranelift_codegen::ir::{ + types, AbiParam, Block, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc, + StackSlot, StackSlotData, StackSlotKind, TrapCode, Type, Value, + }; + pub(crate) use cranelift_codegen::Context; + pub(crate) use cranelift_module::{self, DataDescription, FuncId, Linkage, Module}; + pub(crate) use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; + pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE}; + pub(crate) use rustc_index::Idx; + pub(crate) use rustc_middle::mir::{self, *}; + pub(crate) use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; + pub(crate) use rustc_middle::ty::{ + self, FloatTy, Instance, InstanceKind, IntTy, ParamEnv, Ty, TyCtxt, UintTy, + }; + pub(crate) use rustc_span::Span; + pub(crate) use rustc_target::abi::{Abi, FieldIdx, Scalar, Size, VariantIdx, FIRST_VARIANT}; + + pub(crate) use crate::abi::*; + pub(crate) use crate::base::{codegen_operand, codegen_place}; + pub(crate) use crate::cast::*; + pub(crate) use crate::common::*; + pub(crate) use crate::debuginfo::{DebugContext, UnwindContext}; + pub(crate) use crate::pointer::Pointer; + pub(crate) use crate::value_and_place::{CPlace, CValue}; +} + +struct PrintOnPanic<F: Fn() -> String>(F); +impl<F: Fn() -> String> Drop for PrintOnPanic<F> { + fn drop(&mut self) { + if ::std::thread::panicking() { + println!("{}", (self.0)()); + } + } +} + +/// The codegen context holds any information shared between the codegen of individual functions +/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module). +struct CodegenCx { + profiler: SelfProfilerRef, + output_filenames: Arc<OutputFilenames>, + should_write_ir: bool, + global_asm: String, + inline_asm_index: Cell<usize>, + debug_context: Option<DebugContext>, + cgu_name: Symbol, +} + +impl CodegenCx { + fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, debug_info: bool, cgu_name: Symbol) -> Self { + assert_eq!(pointer_ty(tcx), isa.pointer_type()); + + let debug_context = if debug_info && !tcx.sess.target.options.is_like_windows { + Some(DebugContext::new(tcx, isa, cgu_name.as_str())) + } else { + None + }; + CodegenCx { + profiler: tcx.prof.clone(), + output_filenames: tcx.output_filenames(()).clone(), + should_write_ir: crate::pretty_clif::should_write_ir(tcx), + global_asm: String::new(), + inline_asm_index: Cell::new(0), + debug_context, + cgu_name, + } + } +} + +pub struct CraneliftCodegenBackend { + pub config: RefCell<Option<BackendConfig>>, +} + +impl CodegenBackend for CraneliftCodegenBackend { + fn locale_resource(&self) -> &'static str { + // FIXME(rust-lang/rust#100717) - cranelift codegen backend is not yet translated + "" + } + + fn init(&self, sess: &Session) { + use rustc_session::config::{InstrumentCoverage, Lto}; + match sess.lto() { + Lto::No | Lto::ThinLocal => {} + Lto::Thin | Lto::Fat => { + sess.dcx().warn("LTO is not supported. You may get a linker error.") + } + } + + if sess.opts.cg.instrument_coverage() != InstrumentCoverage::No { + sess.dcx() + .fatal("`-Cinstrument-coverage` is LLVM specific and not supported by Cranelift"); + } + + let mut config = self.config.borrow_mut(); + if config.is_none() { + let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args) + .unwrap_or_else(|err| sess.dcx().fatal(err)); + *config = Some(new_config); + } + } + + fn target_features(&self, sess: &Session, _allow_unstable: bool) -> Vec<rustc_span::Symbol> { + // FIXME return the actually used target features. this is necessary for #[cfg(target_feature)] + if sess.target.arch == "x86_64" && sess.target.os != "none" { + // x86_64 mandates SSE2 support + vec![Symbol::intern("fxsr"), sym::sse, Symbol::intern("sse2")] + } else if sess.target.arch == "aarch64" { + match &*sess.target.os { + "none" => vec![], + // On macOS the aes, sha2 and sha3 features are enabled by default and ring + // fails to compile on macOS when they are not present. + "macos" => vec![ + sym::neon, + Symbol::intern("aes"), + Symbol::intern("sha2"), + Symbol::intern("sha3"), + ], + // AArch64 mandates Neon support + _ => vec![sym::neon], + } + } else { + vec![] + } + } + + fn print_version(&self) { + println!("Cranelift version: {}", cranelift_codegen::VERSION); + } + + fn codegen_crate( + &self, + tcx: TyCtxt<'_>, + metadata: EncodedMetadata, + need_metadata_module: bool, + ) -> Box<dyn Any> { + tcx.dcx().abort_if_errors(); + let config = self.config.borrow().clone().unwrap(); + match config.codegen_mode { + CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), + CodegenMode::Jit | CodegenMode::JitLazy => { + #[cfg(feature = "jit")] + driver::jit::run_jit(tcx, config); + + #[cfg(not(feature = "jit"))] + tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } + } + } + + fn join_codegen( + &self, + ongoing_codegen: Box<dyn Any>, + sess: &Session, + outputs: &OutputFilenames, + ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) { + ongoing_codegen.downcast::<driver::aot::OngoingCodegen>().unwrap().join( + sess, + outputs, + self.config.borrow().as_ref().unwrap(), + ) + } + + fn link( + &self, + sess: &Session, + codegen_results: CodegenResults, + outputs: &OutputFilenames, + ) -> Result<(), ErrorGuaranteed> { + use rustc_codegen_ssa::back::link::link_binary; + + link_binary(sess, &crate::archive::ArArchiveBuilderBuilder, &codegen_results, outputs) + } +} + +fn target_triple(sess: &Session) -> target_lexicon::Triple { + match sess.target.llvm_target.parse() { + Ok(triple) => triple, + Err(err) => sess.dcx().fatal(format!("target not recognized: {}", err)), + } +} + +fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn TargetIsa + 'static> { + use target_lexicon::BinaryFormat; + + let target_triple = crate::target_triple(sess); + + let mut flags_builder = settings::builder(); + flags_builder.enable("is_pic").unwrap(); + let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" }; + flags_builder.set("enable_verifier", enable_verifier).unwrap(); + flags_builder.set("regalloc_checker", enable_verifier).unwrap(); + + let mut frame_ptr = sess.target.options.frame_pointer.clone(); + frame_ptr.ratchet(sess.opts.cg.force_frame_pointers); + let preserve_frame_pointer = frame_ptr != rustc_target::spec::FramePointer::MayOmit; + flags_builder + .set("preserve_frame_pointers", if preserve_frame_pointer { "true" } else { "false" }) + .unwrap(); + + let tls_model = match target_triple.binary_format { + BinaryFormat::Elf => "elf_gd", + BinaryFormat::Macho => "macho", + BinaryFormat::Coff => "coff", + _ => "none", + }; + flags_builder.set("tls_model", tls_model).unwrap(); + + flags_builder.set("enable_llvm_abi_extensions", "true").unwrap(); + + use rustc_session::config::OptLevel; + match sess.opts.optimize { + OptLevel::No => { + flags_builder.set("opt_level", "none").unwrap(); + } + OptLevel::Less | OptLevel::Default => {} + OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => { + flags_builder.set("opt_level", "speed_and_size").unwrap(); + } + } + + if let target_lexicon::Architecture::Aarch64(_) + | target_lexicon::Architecture::Riscv64(_) + | target_lexicon::Architecture::X86_64 = target_triple.architecture + { + // Windows depends on stack probes to grow the committed part of the stack. + // On other platforms it helps prevents stack smashing. + flags_builder.enable("enable_probestack").unwrap(); + flags_builder.set("probestack_strategy", "inline").unwrap(); + } else { + // __cranelift_probestack is not provided and inline stack probes are only supported on + // AArch64, Riscv64 and x86_64. + flags_builder.set("enable_probestack", "false").unwrap(); + } + + let flags = settings::Flags::new(flags_builder); + + let isa_builder = match sess.opts.cg.target_cpu.as_deref() { + Some("native") => cranelift_native::builder_with_options(true).unwrap(), + Some(value) => { + let mut builder = + cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| { + sess.dcx().fatal(format!("can't compile for {}: {}", target_triple, err)); + }); + if builder.enable(value).is_err() { + sess.dcx() + .fatal("the specified target cpu isn't currently supported by Cranelift."); + } + builder + } + None => { + let mut builder = + cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| { + sess.dcx().fatal(format!("can't compile for {}: {}", target_triple, err)); + }); + if target_triple.architecture == target_lexicon::Architecture::X86_64 { + // Only set the target cpu on x86_64 as Cranelift is missing + // the target cpu list for most other targets. + builder.enable(sess.target.cpu.as_ref()).unwrap(); + } + builder + } + }; + + match isa_builder.finish(flags) { + Ok(target_isa) => target_isa, + Err(err) => sess.dcx().fatal(format!("failed to build TargetIsa: {}", err)), + } +} + +/// This is the entrypoint for a hot plugged rustc_codegen_cranelift +#[no_mangle] +pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> { + Box::new(CraneliftCodegenBackend { config: RefCell::new(None) }) +} diff --git a/compiler/rustc_codegen_cranelift/src/linkage.rs b/compiler/rustc_codegen_cranelift/src/linkage.rs new file mode 100644 index 00000000000..ca853aac158 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/linkage.rs @@ -0,0 +1,36 @@ +use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; + +use crate::prelude::*; + +pub(crate) fn get_clif_linkage( + mono_item: MonoItem<'_>, + linkage: RLinkage, + visibility: Visibility, + is_compiler_builtins: bool, +) -> Linkage { + match (linkage, visibility) { + (RLinkage::External, Visibility::Default) if is_compiler_builtins => Linkage::Hidden, + (RLinkage::External, Visibility::Default) => Linkage::Export, + (RLinkage::Internal, Visibility::Default) => Linkage::Local, + (RLinkage::External, Visibility::Hidden) => Linkage::Hidden, + (RLinkage::WeakAny, Visibility::Default) => Linkage::Preemptible, + _ => panic!("{:?} = {:?} {:?}", mono_item, linkage, visibility), + } +} + +pub(crate) fn get_static_linkage(tcx: TyCtxt<'_>, def_id: DefId) -> Linkage { + let fn_attrs = tcx.codegen_fn_attrs(def_id); + + if let Some(linkage) = fn_attrs.linkage { + match linkage { + RLinkage::External => Linkage::Export, + RLinkage::Internal => Linkage::Local, + RLinkage::ExternalWeak | RLinkage::WeakAny => Linkage::Preemptible, + _ => panic!("{:?}", linkage), + } + } else if tcx.is_reachable_non_generic(def_id) { + Linkage::Export + } else { + Linkage::Hidden + } +} diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs new file mode 100644 index 00000000000..ba20a750d2b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs @@ -0,0 +1,174 @@ +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use rustc_hir::LangItem; +use rustc_middle::ty::{AssocKind, GenericArg}; +use rustc_session::config::{sigpipe, EntryFnType}; +use rustc_span::symbol::Ident; +use rustc_span::DUMMY_SP; + +use crate::prelude::*; + +/// Create the `main` function which will initialize the rust runtime and call +/// users main function. +pub(crate) fn maybe_create_entry_wrapper( + tcx: TyCtxt<'_>, + module: &mut dyn Module, + is_jit: bool, + is_primary_cgu: bool, +) { + let (main_def_id, (is_main_fn, sigpipe)) = match tcx.entry_fn(()) { + Some((def_id, entry_ty)) => ( + def_id, + match entry_ty { + EntryFnType::Main { sigpipe } => (true, sigpipe), + EntryFnType::Start => (false, sigpipe::DEFAULT), + }, + ), + None => return, + }; + + if main_def_id.is_local() { + let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); + if module.get_name(tcx.symbol_name(instance).name).is_none() { + return; + } + } else if !is_primary_cgu { + return; + } + + create_entry_fn(tcx, module, main_def_id, is_jit, is_main_fn, sigpipe); + + fn create_entry_fn( + tcx: TyCtxt<'_>, + m: &mut dyn Module, + rust_main_def_id: DefId, + ignore_lang_start_wrapper: bool, + is_main_fn: bool, + sigpipe: u8, + ) { + let main_ret_ty = tcx.fn_sig(rust_main_def_id).no_bound_vars().unwrap().output(); + // Given that `main()` has no arguments, + // then its return type cannot have + // late-bound regions, since late-bound + // regions must appear in the argument + // listing. + let main_ret_ty = tcx.normalize_erasing_regions( + ty::ParamEnv::reveal_all(), + main_ret_ty.no_bound_vars().unwrap(), + ); + + let cmain_sig = Signature { + params: vec![ + AbiParam::new(m.target_config().pointer_type()), + AbiParam::new(m.target_config().pointer_type()), + ], + returns: vec![AbiParam::new(m.target_config().pointer_type() /*isize*/)], + call_conv: crate::conv_to_call_conv( + tcx.sess, + tcx.sess.target.options.entry_abi, + m.target_config().default_call_conv, + ), + }; + + let entry_name = tcx.sess.target.options.entry_name.as_ref(); + let cmain_func_id = match m.declare_function(entry_name, Linkage::Export, &cmain_sig) { + Ok(func_id) => func_id, + Err(err) => { + tcx.dcx() + .fatal(format!("entry symbol `{entry_name}` declared multiple times: {err}")); + } + }; + + let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); + + let main_name = tcx.symbol_name(instance).name; + let main_sig = get_function_sig(tcx, m.target_config().default_call_conv, instance); + let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap(); + + let mut ctx = Context::new(); + ctx.func.signature = cmain_sig; + { + let mut func_ctx = FunctionBuilderContext::new(); + let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let block = bcx.create_block(); + bcx.switch_to_block(block); + let arg_argc = bcx.append_block_param(block, m.target_config().pointer_type()); + let arg_argv = bcx.append_block_param(block, m.target_config().pointer_type()); + let arg_sigpipe = bcx.ins().iconst(types::I8, sigpipe as i64); + + let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func); + + let result = if is_main_fn && ignore_lang_start_wrapper { + // regular main fn, but ignoring #[lang = "start"] as we are running in the jit + // FIXME set program arguments somehow + let call_inst = bcx.ins().call(main_func_ref, &[]); + let call_results = bcx.func.dfg.inst_results(call_inst).to_owned(); + + let termination_trait = tcx.require_lang_item(LangItem::Termination, None); + let report = tcx + .associated_items(termination_trait) + .find_by_name_and_kind( + tcx, + Ident::from_str("report"), + AssocKind::Fn, + termination_trait, + ) + .unwrap(); + let report = Instance::expect_resolve( + tcx, + ParamEnv::reveal_all(), + report.def_id, + tcx.mk_args(&[GenericArg::from(main_ret_ty)]), + DUMMY_SP, + ) + .polymorphize(tcx); + + let report_name = tcx.symbol_name(report).name; + let report_sig = get_function_sig(tcx, m.target_config().default_call_conv, report); + let report_func_id = + m.declare_function(report_name, Linkage::Import, &report_sig).unwrap(); + let report_func_ref = m.declare_func_in_func(report_func_id, &mut bcx.func); + + // FIXME do proper abi handling instead of expecting the pass mode to be identical + // for returns and arguments. + let report_call_inst = bcx.ins().call(report_func_ref, &call_results); + let res = bcx.func.dfg.inst_results(report_call_inst)[0]; + match m.target_config().pointer_type() { + types::I32 => res, + types::I64 => bcx.ins().sextend(types::I64, res), + _ => unimplemented!("16bit systems are not yet supported"), + } + } else if is_main_fn { + let start_def_id = tcx.require_lang_item(LangItem::Start, None); + let start_instance = Instance::expect_resolve( + tcx, + ParamEnv::reveal_all(), + start_def_id, + tcx.mk_args(&[main_ret_ty.into()]), + DUMMY_SP, + ) + .polymorphize(tcx); + let start_func_id = import_function(tcx, m, start_instance); + + let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref); + + let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func); + let call_inst = + bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv, arg_sigpipe]); + bcx.inst_results(call_inst)[0] + } else { + // using user-defined start fn + let call_inst = bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] + }; + + bcx.ins().return_(&[result]); + bcx.seal_all_blocks(); + bcx.finalize(); + } + + if let Err(err) = m.define_function(cmain_func_id, &mut ctx) { + tcx.dcx().fatal(format!("entry symbol `{entry_name}` defined multiple times: {err}")); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs new file mode 100644 index 00000000000..fb18f45d7dc --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/num.rs @@ -0,0 +1,467 @@ +//! Various operations on integer and floating-point numbers + +use crate::prelude::*; + +pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> { + use BinOp::*; + use IntCC::*; + Some(match bin_op { + Eq => Equal, + Lt => { + if signed { + SignedLessThan + } else { + UnsignedLessThan + } + } + Le => { + if signed { + SignedLessThanOrEqual + } else { + UnsignedLessThanOrEqual + } + } + Ne => NotEqual, + Ge => { + if signed { + SignedGreaterThanOrEqual + } else { + UnsignedGreaterThanOrEqual + } + } + Gt => { + if signed { + SignedGreaterThan + } else { + UnsignedGreaterThan + } + } + _ => return None, + }) +} + +fn codegen_three_way_compare<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + signed: bool, + lhs: Value, + rhs: Value, +) -> CValue<'tcx> { + // This emits `(lhs > rhs) - (lhs < rhs)`, which is cranelift's preferred form per + // <https://github.com/bytecodealliance/wasmtime/blob/8052bb9e3b792503b225f2a5b2ba3bc023bff462/cranelift/codegen/src/prelude_opt.isle#L41-L47> + let gt_cc = crate::num::bin_op_to_intcc(BinOp::Gt, signed).unwrap(); + let lt_cc = crate::num::bin_op_to_intcc(BinOp::Lt, signed).unwrap(); + let gt = fx.bcx.ins().icmp(gt_cc, lhs, rhs); + let lt = fx.bcx.ins().icmp(lt_cc, lhs, rhs); + let val = fx.bcx.ins().isub(gt, lt); + CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(Some(fx.mir.span)))) +} + +fn codegen_compare_bin_op<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + signed: bool, + lhs: Value, + rhs: Value, +) -> CValue<'tcx> { + if bin_op == BinOp::Cmp { + return codegen_three_way_compare(fx, signed, lhs, rhs); + } + + let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap(); + let val = fx.bcx.ins().icmp(intcc, lhs, rhs); + CValue::by_val(val, fx.layout_of(fx.tcx.types.bool)) +} + +pub(crate) fn codegen_binop<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + match bin_op { + BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt | BinOp::Cmp => { + match in_lhs.layout().ty.kind() { + ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => { + let signed = type_sign(in_lhs.layout().ty); + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + return codegen_compare_bin_op(fx, bin_op, signed, lhs, rhs); + } + _ => {} + } + } + _ => {} + } + + match in_lhs.layout().ty.kind() { + ty::Bool => crate::num::codegen_bool_binop(fx, bin_op, in_lhs, in_rhs), + ty::Uint(_) | ty::Int(_) => crate::num::codegen_int_binop(fx, bin_op, in_lhs, in_rhs), + ty::Float(_) => crate::num::codegen_float_binop(fx, bin_op, in_lhs, in_rhs), + ty::RawPtr(..) | ty::FnPtr(..) => crate::num::codegen_ptr_binop(fx, bin_op, in_lhs, in_rhs), + _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty), + } +} + +pub(crate) fn codegen_bool_binop<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + let b = fx.bcx.ins(); + let res = match bin_op { + BinOp::BitXor => b.bxor(lhs, rhs), + BinOp::BitAnd => b.band(lhs, rhs), + BinOp::BitOr => b.bor(lhs, rhs), + // Compare binops handles by `codegen_binop`. + _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs), + }; + + CValue::by_val(res, fx.layout_of(fx.tcx.types.bool)) +} + +pub(crate) fn codegen_int_binop<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + if !matches!(bin_op, BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked) { + assert_eq!( + in_lhs.layout().ty, + in_rhs.layout().ty, + "int binop requires lhs and rhs of same type" + ); + } + + if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, in_lhs, in_rhs) { + return res; + } + + let signed = type_sign(in_lhs.layout().ty); + + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + let b = fx.bcx.ins(); + // FIXME trap on overflow for the Unchecked versions + let val = match bin_op { + BinOp::Add | BinOp::AddUnchecked => b.iadd(lhs, rhs), + BinOp::Sub | BinOp::SubUnchecked => b.isub(lhs, rhs), + BinOp::Mul | BinOp::MulUnchecked => b.imul(lhs, rhs), + BinOp::Div => { + if signed { + b.sdiv(lhs, rhs) + } else { + b.udiv(lhs, rhs) + } + } + BinOp::Rem => { + if signed { + b.srem(lhs, rhs) + } else { + b.urem(lhs, rhs) + } + } + BinOp::BitXor => b.bxor(lhs, rhs), + BinOp::BitAnd => b.band(lhs, rhs), + BinOp::BitOr => b.bor(lhs, rhs), + BinOp::Shl | BinOp::ShlUnchecked => b.ishl(lhs, rhs), + BinOp::Shr | BinOp::ShrUnchecked => { + if signed { + b.sshr(lhs, rhs) + } else { + b.ushr(lhs, rhs) + } + } + BinOp::Offset => unreachable!("Offset is not an integer operation"), + BinOp::AddWithOverflow | BinOp::SubWithOverflow | BinOp::MulWithOverflow => { + unreachable!("Overflow binops handled by `codegen_checked_int_binop`") + } + // Compare binops handles by `codegen_binop`. + BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge | BinOp::Cmp => { + unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty); + } + }; + + CValue::by_val(val, in_lhs.layout()) +} + +pub(crate) fn codegen_checked_int_binop<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + if let Some(res) = crate::codegen_i128::maybe_codegen_checked(fx, bin_op, in_lhs, in_rhs) { + return res; + } + + let signed = type_sign(in_lhs.layout().ty); + + let (res, has_overflow) = match bin_op { + BinOp::Add => { + /*let (val, c_out) = fx.bcx.ins().iadd_cout(lhs, rhs); + (val, c_out)*/ + // FIXME(CraneStation/cranelift#849) legalize iadd_cout for i8 and i16 + let val = fx.bcx.ins().iadd(lhs, rhs); + let has_overflow = if !signed { + fx.bcx.ins().icmp(IntCC::UnsignedLessThan, val, lhs) + } else { + let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0); + let slt = fx.bcx.ins().icmp(IntCC::SignedLessThan, val, lhs); + fx.bcx.ins().bxor(rhs_is_negative, slt) + }; + (val, has_overflow) + } + BinOp::Sub => { + /*let (val, b_out) = fx.bcx.ins().isub_bout(lhs, rhs); + (val, b_out)*/ + // FIXME(CraneStation/cranelift#849) legalize isub_bout for i8 and i16 + let val = fx.bcx.ins().isub(lhs, rhs); + let has_overflow = if !signed { + fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, val, lhs) + } else { + let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0); + let sgt = fx.bcx.ins().icmp(IntCC::SignedGreaterThan, val, lhs); + fx.bcx.ins().bxor(rhs_is_negative, sgt) + }; + (val, has_overflow) + } + BinOp::Mul => { + let ty = fx.bcx.func.dfg.value_type(lhs); + match ty { + types::I8 | types::I16 | types::I32 if !signed => { + let lhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), lhs); + let rhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), rhs); + let val = fx.bcx.ins().imul(lhs, rhs); + let has_overflow = fx.bcx.ins().icmp_imm( + IntCC::UnsignedGreaterThan, + val, + (1 << ty.bits()) - 1, + ); + let val = fx.bcx.ins().ireduce(ty, val); + (val, has_overflow) + } + types::I8 | types::I16 | types::I32 if signed => { + let lhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), lhs); + let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs); + let val = fx.bcx.ins().imul(lhs, rhs); + let has_underflow = + fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1))); + let has_overflow = fx.bcx.ins().icmp_imm( + IntCC::SignedGreaterThan, + val, + (1 << (ty.bits() - 1)) - 1, + ); + let val = fx.bcx.ins().ireduce(ty, val); + (val, fx.bcx.ins().bor(has_underflow, has_overflow)) + } + types::I64 => { + let val = fx.bcx.ins().imul(lhs, rhs); + let has_overflow = if !signed { + let val_hi = fx.bcx.ins().umulhi(lhs, rhs); + fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0) + } else { + // Based on LLVM's instruction sequence for compiling + // a.checked_mul(b).is_some() to riscv64gc: + // mulh a2, a0, a1 + // mul a0, a0, a1 + // srai a0, a0, 63 + // xor a0, a0, a2 + // snez a0, a0 + let val_hi = fx.bcx.ins().smulhi(lhs, rhs); + let val_sign = fx.bcx.ins().sshr_imm(val, i64::from(ty.bits() - 1)); + let xor = fx.bcx.ins().bxor(val_hi, val_sign); + fx.bcx.ins().icmp_imm(IntCC::NotEqual, xor, 0) + }; + (val, has_overflow) + } + types::I128 => { + unreachable!("i128 should have been handled by codegen_i128::maybe_codegen") + } + _ => unreachable!("invalid non-integer type {}", ty), + } + } + _ => bug!("binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", bin_op, in_lhs, in_rhs), + }; + + let out_layout = fx.layout_of(Ty::new_tup(fx.tcx, &[in_lhs.layout().ty, fx.tcx.types.bool])); + CValue::by_val_pair(res, has_overflow, out_layout) +} + +pub(crate) fn codegen_saturating_int_binop<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + lhs: CValue<'tcx>, + rhs: CValue<'tcx>, +) -> CValue<'tcx> { + assert_eq!(lhs.layout().ty, rhs.layout().ty); + + let signed = type_sign(lhs.layout().ty); + let clif_ty = fx.clif_type(lhs.layout().ty).unwrap(); + let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed); + + let checked_res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs); + let (val, has_overflow) = checked_res.load_scalar_pair(fx); + + let val = match (bin_op, signed) { + (BinOp::Add, false) => fx.bcx.ins().select(has_overflow, max, val), + (BinOp::Sub, false) => fx.bcx.ins().select(has_overflow, min, val), + (BinOp::Add, true) => { + let rhs = rhs.load_scalar(fx); + let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); + fx.bcx.ins().select(has_overflow, sat_val, val) + } + (BinOp::Sub, true) => { + let rhs = rhs.load_scalar(fx); + let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); + fx.bcx.ins().select(has_overflow, sat_val, val) + } + _ => unreachable!(), + }; + + CValue::by_val(val, lhs.layout()) +} + +pub(crate) fn codegen_float_binop<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + assert_eq!(in_lhs.layout().ty, in_rhs.layout().ty); + + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + let b = fx.bcx.ins(); + let res = match bin_op { + BinOp::Add => b.fadd(lhs, rhs), + BinOp::Sub => b.fsub(lhs, rhs), + BinOp::Mul => b.fmul(lhs, rhs), + BinOp::Div => b.fdiv(lhs, rhs), + BinOp::Rem => { + let (name, ty) = match in_lhs.layout().ty.kind() { + ty::Float(FloatTy::F32) => ("fmodf", types::F32), + ty::Float(FloatTy::F64) => ("fmod", types::F64), + _ => bug!(), + }; + + let ret_val = fx.lib_call( + name, + vec![AbiParam::new(ty), AbiParam::new(ty)], + vec![AbiParam::new(ty)], + &[lhs, rhs], + )[0]; + + return CValue::by_val(ret_val, in_lhs.layout()); + } + BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => { + let fltcc = match bin_op { + BinOp::Eq => FloatCC::Equal, + BinOp::Lt => FloatCC::LessThan, + BinOp::Le => FloatCC::LessThanOrEqual, + BinOp::Ne => FloatCC::NotEqual, + BinOp::Ge => FloatCC::GreaterThanOrEqual, + BinOp::Gt => FloatCC::GreaterThan, + _ => unreachable!(), + }; + let val = fx.bcx.ins().fcmp(fltcc, lhs, rhs); + return CValue::by_val(val, fx.layout_of(fx.tcx.types.bool)); + } + _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs), + }; + + CValue::by_val(res, in_lhs.layout()) +} + +pub(crate) fn codegen_ptr_binop<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + let is_thin_ptr = + in_lhs.layout().ty.builtin_deref(true).map(|ty| !has_ptr_meta(fx.tcx, ty)).unwrap_or(true); + + if is_thin_ptr { + match bin_op { + BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => { + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + codegen_compare_bin_op(fx, bin_op, false, lhs, rhs) + } + BinOp::Offset => { + let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap(); + let (base, offset) = (in_lhs, in_rhs.load_scalar(fx)); + let pointee_size = fx.layout_of(pointee_ty).size.bytes(); + let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64); + let base_val = base.load_scalar(fx); + let res = fx.bcx.ins().iadd(base_val, ptr_diff); + CValue::by_val(res, base.layout()) + } + _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs), + } + } else { + let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx); + let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx); + + let res = match bin_op { + BinOp::Eq => { + let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr); + let extra_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_extra, rhs_extra); + fx.bcx.ins().band(ptr_eq, extra_eq) + } + BinOp::Ne => { + let ptr_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_ptr, rhs_ptr); + let extra_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_extra, rhs_extra); + fx.bcx.ins().bor(ptr_ne, extra_ne) + } + BinOp::Lt | BinOp::Le | BinOp::Ge | BinOp::Gt => { + let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr); + + let ptr_cmp = + fx.bcx.ins().icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr); + let extra_cmp = fx.bcx.ins().icmp( + bin_op_to_intcc(bin_op, false).unwrap(), + lhs_extra, + rhs_extra, + ); + + fx.bcx.ins().select(ptr_eq, extra_cmp, ptr_cmp) + } + _ => panic!("bin_op {:?} on ptr", bin_op), + }; + + CValue::by_val(res, fx.layout_of(fx.tcx.types.bool)) + } +} + +// In Rust floating point min and max don't propagate NaN. In Cranelift they do however. +// For this reason it is necessary to use `a.is_nan() ? b : (a >= b ? b : a)` for `minnumf*` +// and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by comparing +// a float against itself. Only in case of NaN is it not equal to itself. +pub(crate) fn codegen_float_min(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { + let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a); + let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b); + let temp = fx.bcx.ins().select(a_ge_b, b, a); + fx.bcx.ins().select(a_is_nan, b, temp) +} + +pub(crate) fn codegen_float_max(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { + let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a); + let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b); + let temp = fx.bcx.ins().select(a_le_b, b, a); + fx.bcx.ins().select(a_is_nan, b, temp) +} diff --git a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs new file mode 100644 index 00000000000..0df7e82294b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs @@ -0,0 +1,3 @@ +//! Various optimizations specific to cg_clif + +pub(crate) mod peephole; diff --git a/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs new file mode 100644 index 00000000000..c93fe935210 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs @@ -0,0 +1,48 @@ +//! Peephole optimizations that can be performed while creating clif ir. + +use cranelift_codegen::ir::condcodes::IntCC; +use cranelift_codegen::ir::{InstructionData, Opcode, Value, ValueDef}; +use cranelift_frontend::FunctionBuilder; + +/// If the given value was produced by the lowering of `Rvalue::Not` return the input and true, +/// otherwise return the given value and false. +pub(crate) fn maybe_unwrap_bool_not(bcx: &mut FunctionBuilder<'_>, arg: Value) -> (Value, bool) { + if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) { + match bcx.func.dfg.insts[arg_inst] { + // This is the lowering of `Rvalue::Not` + InstructionData::IntCompareImm { + opcode: Opcode::IcmpImm, + cond: IntCC::Equal, + arg, + imm, + } if imm.bits() == 0 => (arg, true), + _ => (arg, false), + } + } else { + (arg, false) + } +} + +/// Returns whether the branch is statically known to be taken or `None` if it isn't statically known. +pub(crate) fn maybe_known_branch_taken( + bcx: &FunctionBuilder<'_>, + arg: Value, + test_zero: bool, +) -> Option<bool> { + let arg_inst = if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) { + arg_inst + } else { + return None; + }; + + match bcx.func.dfg.insts[arg_inst] { + InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => { + if test_zero { + Some(imm.bits() == 0) + } else { + Some(imm.bits() != 0) + } + } + _ => None, + } +} diff --git a/compiler/rustc_codegen_cranelift/src/pointer.rs b/compiler/rustc_codegen_cranelift/src/pointer.rs new file mode 100644 index 00000000000..11ac6b94678 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/pointer.rs @@ -0,0 +1,128 @@ +//! Defines [`Pointer`] which is used to improve the quality of the generated clif ir for pointer +//! operations. + +use cranelift_codegen::ir::immediates::Offset32; +use rustc_target::abi::Align; + +use crate::prelude::*; + +/// A pointer pointing either to a certain address, a certain stack slot or nothing. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Pointer { + base: PointerBase, + offset: Offset32, +} + +#[derive(Copy, Clone, Debug)] +pub(crate) enum PointerBase { + Addr(Value), + Stack(StackSlot), + Dangling(Align), +} + +impl Pointer { + pub(crate) fn new(addr: Value) -> Self { + Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) } + } + + pub(crate) fn stack_slot(stack_slot: StackSlot) -> Self { + Pointer { base: PointerBase::Stack(stack_slot), offset: Offset32::new(0) } + } + + pub(crate) fn dangling(align: Align) -> Self { + Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) } + } + + pub(crate) fn debug_base_and_offset(self) -> (PointerBase, Offset32) { + (self.base, self.offset) + } + + pub(crate) fn get_addr(self, fx: &mut FunctionCx<'_, '_, '_>) -> Value { + match self.base { + PointerBase::Addr(base_addr) => { + let offset: i64 = self.offset.into(); + if offset == 0 { base_addr } else { fx.bcx.ins().iadd_imm(base_addr, offset) } + } + PointerBase::Stack(stack_slot) => { + fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, self.offset) + } + PointerBase::Dangling(align) => { + fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()) + } + } + } + + pub(crate) fn offset(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: Offset32) -> Self { + self.offset_i64(fx, extra_offset.into()) + } + + pub(crate) fn offset_i64(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: i64) -> Self { + if let Some(new_offset) = self.offset.try_add_i64(extra_offset) { + Pointer { base: self.base, offset: new_offset } + } else { + let base_offset: i64 = self.offset.into(); + if let Some(new_offset) = base_offset.checked_add(extra_offset) { + let base_addr = match self.base { + PointerBase::Addr(addr) => addr, + PointerBase::Stack(stack_slot) => { + fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0) + } + PointerBase::Dangling(align) => { + fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()) + } + }; + let addr = fx.bcx.ins().iadd_imm(base_addr, new_offset); + Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) } + } else { + panic!( + "self.offset ({}) + extra_offset ({}) not representable in i64", + base_offset, extra_offset + ); + } + } + } + + pub(crate) fn offset_value(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: Value) -> Self { + match self.base { + PointerBase::Addr(addr) => Pointer { + base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)), + offset: self.offset, + }, + PointerBase::Stack(stack_slot) => { + let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, self.offset); + Pointer { + base: PointerBase::Addr(fx.bcx.ins().iadd(base_addr, extra_offset)), + offset: Offset32::new(0), + } + } + PointerBase::Dangling(align) => { + let addr = + fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()); + Pointer { + base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)), + offset: self.offset, + } + } + } + } + + pub(crate) fn load(self, fx: &mut FunctionCx<'_, '_, '_>, ty: Type, flags: MemFlags) -> Value { + match self.base { + PointerBase::Addr(base_addr) => fx.bcx.ins().load(ty, flags, base_addr, self.offset), + PointerBase::Stack(stack_slot) => fx.bcx.ins().stack_load(ty, stack_slot, self.offset), + PointerBase::Dangling(_align) => unreachable!(), + } + } + + pub(crate) fn store(self, fx: &mut FunctionCx<'_, '_, '_>, value: Value, flags: MemFlags) { + match self.base { + PointerBase::Addr(base_addr) => { + fx.bcx.ins().store(flags, value, base_addr, self.offset); + } + PointerBase::Stack(stack_slot) => { + fx.bcx.ins().stack_store(value, stack_slot, self.offset); + } + PointerBase::Dangling(_align) => unreachable!(), + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs new file mode 100644 index 00000000000..196418023d9 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -0,0 +1,281 @@ +//! This module provides the [CommentWriter] which makes it possible +//! to add comments to the written cranelift ir. +//! +//! # Example +//! +//! ```clif +//! test compile +//! target x86_64 +//! +//! function u0:22(i64) -> i8, i8 system_v { +//! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd517c453d67c0915E +//! ; instance Instance { def: Item(WithOptConstParam { did: DefId(0:42 ~ example[4e51]::{impl#0}::call_once), const_param_did: None }), args: [ReErased, ReErased] } +//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: Aggregate { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, abi: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), variants: Single { index: 0 } } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } +//! +//! ; kind loc.idx param pass mode ty +//! ; ssa _0 (u8, u8) 2b 1, 8 var=(0, 1) +//! ; ret _0 - Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) (u8, u8) +//! ; arg _1 - Ignore IsNotEmpty +//! ; arg _2.0 = v0 Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) &&[u16] +//! +//! ; kind local ty size align (abi,pref) +//! ; zst _1 IsNotEmpty 0b 1, 8 align=8,offset= +//! ; stack _2 (&&[u16],) 8b 8, 8 storage=ss0 +//! ; ssa _3 &mut IsNotEmpty 8b 8, 8 var=2 +//! +//! ss0 = explicit_slot 16 +//! sig0 = (i64, i64) -> i8, i8 system_v +//! fn0 = colocated u0:23 sig0 ; Instance { def: Item(WithOptConstParam { did: DefId(0:46 ~ example[4e51]::{impl#1}::call_mut), const_param_did: None }), args: [ReErased, ReErased] } +//! +//! block0(v0: i64): +//! nop +//! ; write_cvalue: Addr(Pointer { base: Stack(ss0), offset: Offset32(0) }, None): &&[u16] <- ByVal(v0): &&[u16] +//! stack_store v0, ss0 +//! jump block1 +//! +//! block1: +//! nop +//! ; _3 = &mut _1 +//! v1 = iconst.i64 8 +//! ; write_cvalue: Var(_3, var2): &mut IsNotEmpty <- ByVal(v1): &mut IsNotEmpty +//! ; +//! ; _0 = <IsNotEmpty as mini_core::FnMut<(&&[u16],)>>::call_mut(move _3, _2) +//! v2 = stack_load.i64 ss0 +//! v3, v4 = call fn0(v1, v2) ; v1 = 8 +//! v5 -> v3 +//! v6 -> v4 +//! ; write_cvalue: VarPair(_0, var0, var1): (u8, u8) <- ByValPair(v3, v4): (u8, u8) +//! jump block2 +//! +//! block2: +//! nop +//! ; +//! ; return +//! return v5, v6 +//! } +//! ``` + +use std::fmt; +use std::io::Write; + +use cranelift_codegen::entity::SecondaryMap; +use cranelift_codegen::ir::entities::AnyEntity; +use cranelift_codegen::ir::Fact; +use cranelift_codegen::write::{FuncWriter, PlainWriter}; +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_session::config::{OutputFilenames, OutputType}; + +use crate::prelude::*; + +#[derive(Clone, Debug)] +pub(crate) struct CommentWriter { + enabled: bool, + global_comments: Vec<String>, + entity_comments: FxHashMap<AnyEntity, String>, +} + +impl CommentWriter { + pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { + let enabled = should_write_ir(tcx); + let global_comments = if enabled { + with_no_trimmed_paths!({ + vec![ + format!("symbol {}", tcx.symbol_name(instance).name), + format!("instance {:?}", instance), + format!( + "abi {:?}", + RevealAllLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty()) + ), + String::new(), + ] + }) + } else { + vec![] + }; + + CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() } + } +} + +impl CommentWriter { + pub(crate) fn enabled(&self) -> bool { + self.enabled + } + + pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) { + debug_assert!(self.enabled); + self.global_comments.push(comment.into()); + } + + pub(crate) fn add_comment<S: Into<String> + AsRef<str>, E: Into<AnyEntity>>( + &mut self, + entity: E, + comment: S, + ) { + debug_assert!(self.enabled); + + use std::collections::hash_map::Entry; + match self.entity_comments.entry(entity.into()) { + Entry::Occupied(mut occ) => { + occ.get_mut().push('\n'); + occ.get_mut().push_str(comment.as_ref()); + } + Entry::Vacant(vac) => { + vac.insert(comment.into()); + } + } + } +} + +impl FuncWriter for &'_ CommentWriter { + fn write_preamble( + &mut self, + w: &mut dyn fmt::Write, + func: &Function, + ) -> Result<bool, fmt::Error> { + for comment in &self.global_comments { + if !comment.is_empty() { + writeln!(w, "; {}", comment)?; + } else { + writeln!(w)?; + } + } + if !self.global_comments.is_empty() { + writeln!(w)?; + } + + self.super_preamble(w, func) + } + + fn write_entity_definition( + &mut self, + w: &mut dyn fmt::Write, + _func: &Function, + entity: AnyEntity, + value: &dyn fmt::Display, + maybe_fact: Option<&Fact>, + ) -> fmt::Result { + if let Some(fact) = maybe_fact { + write!(w, " {} ! {} = {}", entity, fact, value)?; + } else { + write!(w, " {} = {}", entity, value)?; + } + + if let Some(comment) = self.entity_comments.get(&entity) { + writeln!(w, " ; {}", comment.replace('\n', "\n; ")) + } else { + writeln!(w) + } + } + + fn write_block_header( + &mut self, + w: &mut dyn fmt::Write, + func: &Function, + block: Block, + indent: usize, + ) -> fmt::Result { + PlainWriter.write_block_header(w, func, block, indent) + } + + fn write_instruction( + &mut self, + w: &mut dyn fmt::Write, + func: &Function, + aliases: &SecondaryMap<Value, Vec<Value>>, + inst: Inst, + indent: usize, + ) -> fmt::Result { + PlainWriter.write_instruction(w, func, aliases, inst, indent)?; + if let Some(comment) = self.entity_comments.get(&inst.into()) { + writeln!(w, "; {}", comment.replace('\n', "\n; "))?; + } + Ok(()) + } +} + +impl FunctionCx<'_, '_, '_> { + pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) { + self.clif_comments.add_global_comment(comment); + } + + pub(crate) fn add_comment<S: Into<String> + AsRef<str>, E: Into<AnyEntity>>( + &mut self, + entity: E, + comment: S, + ) { + self.clif_comments.add_comment(entity, comment); + } +} + +pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { + tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly) +} + +pub(crate) fn write_ir_file( + output_filenames: &OutputFilenames, + name: &str, + write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, +) { + let clif_output_dir = output_filenames.with_extension("clif"); + + match std::fs::create_dir(&clif_output_dir) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {} + res @ Err(_) => res.unwrap(), + } + + let clif_file_name = clif_output_dir.join(name); + + let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file)); + if let Err(err) = res { + // Using early_warn as no Session is available here + let handler = + rustc_session::EarlyDiagCtxt::new(rustc_session::config::ErrorOutputType::default()); + handler.early_warn(format!("error writing ir file: {}", err)); + } +} + +pub(crate) fn write_clif_file( + output_filenames: &OutputFilenames, + symbol_name: &str, + postfix: &str, + isa: &dyn cranelift_codegen::isa::TargetIsa, + func: &cranelift_codegen::ir::Function, + mut clif_comments: &CommentWriter, +) { + // FIXME work around filename too long errors + write_ir_file(output_filenames, &format!("{}.{}.clif", symbol_name, postfix), |file| { + let mut clif = String::new(); + cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func).unwrap(); + + for flag in isa.flags().iter() { + writeln!(file, "set {}", flag)?; + } + write!(file, "target {}", isa.triple().architecture)?; + for isa_flag in isa.isa_flags().iter() { + write!(file, " {}", isa_flag)?; + } + writeln!(file, "\n")?; + writeln!(file)?; + file.write_all(clif.as_bytes())?; + Ok(()) + }); +} + +impl fmt::Debug for FunctionCx<'_, '_, '_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{:?}", self.instance.args)?; + writeln!(f, "{:?}", self.local_map)?; + + let mut clif = String::new(); + ::cranelift_codegen::write::decorate_function( + &mut &self.clif_comments, + &mut clif, + &self.bcx.func, + ) + .unwrap(); + writeln!(f, "\n{}", clif) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/toolchain.rs b/compiler/rustc_codegen_cranelift/src/toolchain.rs new file mode 100644 index 00000000000..b6b465e1f4e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/toolchain.rs @@ -0,0 +1,29 @@ +//! Locating various executables part of a C toolchain. + +use std::path::PathBuf; + +use rustc_codegen_ssa::back::link::linker_and_flavor; +use rustc_session::Session; + +/// Tries to infer the path of a binary for the target toolchain from the linker name. +pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf { + let (mut linker, _linker_flavor) = linker_and_flavor(sess); + let linker_file_name = + linker.file_name().unwrap().to_str().expect("linker filename should be valid UTF-8"); + + if linker_file_name == "ld.lld" { + if tool != "ld" { + linker.set_file_name(tool) + } + } else { + let tool_file_name = linker_file_name + .replace("ld", tool) + .replace("gcc", tool) + .replace("clang", tool) + .replace("cc", tool); + + linker.set_file_name(tool_file_name) + } + + linker +} diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs new file mode 100644 index 00000000000..2fb0c2164c3 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/trap.rs @@ -0,0 +1,38 @@ +//! Helpers used to print a message and abort in case of certain panics and some detected UB. + +use crate::prelude::*; + +fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { + let puts = fx + .module + .declare_function( + "puts", + Linkage::Import, + &Signature { + call_conv: fx.target_config.default_call_conv, + params: vec![AbiParam::new(fx.pointer_type)], + returns: vec![AbiParam::new(types::I32)], + }, + ) + .unwrap(); + let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(puts, "puts"); + } + + let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg); + let msg_ptr = fx.anonymous_str(&real_msg); + fx.bcx.ins().call(puts, &[msg_ptr]); +} + +/// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen. +/// +/// Trap code: user65535 +pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) { + codegen_print(fx, msg.as_ref()); + + let one = fx.bcx.ins().iconst(types::I32, 1); + fx.lib_call("exit", vec![AbiParam::new(types::I32)], vec![], &[one]); + + fx.bcx.ins().trap(TrapCode::User(!0)); +} diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs new file mode 100644 index 00000000000..e09cd16e89a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/unsize.rs @@ -0,0 +1,312 @@ +//! Codegen of the [`PointerCoercion::Unsize`] operation. +//! +//! [`PointerCoercion::Unsize`]: `rustc_middle::ty::adjustment::PointerCoercion::Unsize` + +use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; + +use crate::base::codegen_panic_nounwind; +use crate::prelude::*; + +// Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/base.rs#L159-L307 + +/// Retrieve the information we are losing (making dynamic) in an unsizing +/// adjustment. +/// +/// The `old_info` argument is a bit funny. It is intended for use +/// in an upcast, where the new vtable for an object will be derived +/// from the old one. +pub(crate) fn unsized_info<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + source: Ty<'tcx>, + target: Ty<'tcx>, + old_info: Option<Value>, +) -> Value { + let (source, target) = + fx.tcx.struct_lockstep_tails_for_codegen(source, target, ParamEnv::reveal_all()); + match (&source.kind(), &target.kind()) { + (&ty::Array(_, len), &ty::Slice(_)) => fx + .bcx + .ins() + .iconst(fx.pointer_type, len.eval_target_usize(fx.tcx, ParamEnv::reveal_all()) as i64), + (&ty::Dynamic(data_a, _, src_dyn_kind), &ty::Dynamic(data_b, _, target_dyn_kind)) + if src_dyn_kind == target_dyn_kind => + { + let old_info = + old_info.expect("unsized_info: missing old info for trait upcasting coercion"); + if data_a.principal_def_id() == data_b.principal_def_id() { + // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables. + return old_info; + } + + // trait upcasting coercion + let vptr_entry_idx = fx.tcx.supertrait_vtable_slot((source, target)); + + if let Some(entry_idx) = vptr_entry_idx { + let entry_idx = u32::try_from(entry_idx).unwrap(); + let entry_offset = entry_idx * fx.pointer_type.bytes(); + let vptr_ptr = Pointer::new(old_info).offset_i64(fx, entry_offset.into()).load( + fx, + fx.pointer_type, + crate::vtable::vtable_memflags(), + ); + vptr_ptr + } else { + old_info + } + } + (_, ty::Dynamic(data, ..)) => crate::vtable::get_vtable(fx, source, data.principal()), + _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target), + } +} + +/// Coerce `src` to `dst_ty`. +fn unsize_ptr<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + src: Value, + src_layout: TyAndLayout<'tcx>, + dst_layout: TyAndLayout<'tcx>, + old_info: Option<Value>, +) -> (Value, Value) { + match (&src_layout.ty.kind(), &dst_layout.ty.kind()) { + (&ty::Ref(_, a, _), &ty::Ref(_, b, _)) + | (&ty::Ref(_, a, _), &ty::RawPtr(b, _)) + | (&ty::RawPtr(a, _), &ty::RawPtr(b, _)) => (src, unsized_info(fx, *a, *b, old_info)), + (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { + assert_eq!(def_a, def_b); + + if src_layout == dst_layout { + return (src, old_info.unwrap()); + } + + let mut result = None; + for i in 0..src_layout.fields.count() { + let src_f = src_layout.field(fx, i); + assert_eq!(src_layout.fields.offset(i).bytes(), 0); + assert_eq!(dst_layout.fields.offset(i).bytes(), 0); + if src_f.is_1zst() { + // We are looking for the one non-1-ZST field; this is not it. + continue; + } + assert_eq!(src_layout.size, src_f.size); + + let dst_f = dst_layout.field(fx, i); + assert_ne!(src_f.ty, dst_f.ty); + assert_eq!(result, None); + result = Some(unsize_ptr(fx, src, src_f, dst_f, old_info)); + } + result.unwrap() + } + _ => bug!("unsize_ptr: called on bad types"), + } +} + +/// Coerces `src` to `dst_ty` which is guaranteed to be a `dyn*` type. +pub(crate) fn cast_to_dyn_star<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + src: Value, + src_ty_and_layout: TyAndLayout<'tcx>, + dst_ty: Ty<'tcx>, + old_info: Option<Value>, +) -> (Value, Value) { + assert!( + matches!(dst_ty.kind(), ty::Dynamic(_, _, ty::DynStar)), + "destination type must be a dyn*" + ); + (src, unsized_info(fx, src_ty_and_layout.ty, dst_ty, old_info)) +} + +/// Coerce `src`, which is a reference to a value of type `src_ty`, +/// to a value of type `dst_ty` and store the result in `dst` +pub(crate) fn coerce_unsized_into<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + src: CValue<'tcx>, + dst: CPlace<'tcx>, +) { + let src_ty = src.layout().ty; + let dst_ty = dst.layout().ty; + let mut coerce_ptr = || { + let (base, info) = + if fx.layout_of(src.layout().ty.builtin_deref(true).unwrap()).is_unsized() { + let (old_base, old_info) = src.load_scalar_pair(fx); + unsize_ptr(fx, old_base, src.layout(), dst.layout(), Some(old_info)) + } else { + let base = src.load_scalar(fx); + unsize_ptr(fx, base, src.layout(), dst.layout(), None) + }; + dst.write_cvalue(fx, CValue::by_val_pair(base, info, dst.layout())); + }; + match (&src_ty.kind(), &dst_ty.kind()) { + (&ty::Ref(..), &ty::Ref(..)) + | (&ty::Ref(..), &ty::RawPtr(..)) + | (&ty::RawPtr(..), &ty::RawPtr(..)) => coerce_ptr(), + (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { + assert_eq!(def_a, def_b); + + for i in 0..def_a.variant(FIRST_VARIANT).fields.len() { + let src_f = src.value_field(fx, FieldIdx::new(i)); + let dst_f = dst.place_field(fx, FieldIdx::new(i)); + + if dst_f.layout().is_zst() { + // No data here, nothing to copy/coerce. + continue; + } + + if src_f.layout().ty == dst_f.layout().ty { + dst_f.write_cvalue(fx, src_f); + } else { + coerce_unsized_into(fx, src_f, dst_f); + } + } + } + _ => bug!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty, dst_ty), + } +} + +pub(crate) fn coerce_dyn_star<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + src: CValue<'tcx>, + dst: CPlace<'tcx>, +) { + let (data, extra) = if let ty::Dynamic(_, _, ty::DynStar) = src.layout().ty.kind() { + let (data, vtable) = src.load_scalar_pair(fx); + (data, Some(vtable)) + } else { + let data = src.load_scalar(fx); + (data, None) + }; + + let (data, vtable) = cast_to_dyn_star(fx, data, src.layout(), dst.layout().ty, extra); + + dst.write_cvalue(fx, CValue::by_val_pair(data, vtable, dst.layout())); +} + +// Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/glue.rs + +pub(crate) fn size_and_align_of<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + layout: TyAndLayout<'tcx>, + info: Option<Value>, +) -> (Value, Value) { + if layout.is_sized() { + return ( + fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64), + fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64), + ); + } + + let ty = layout.ty; + match ty.kind() { + ty::Dynamic(..) => { + // load size/align from vtable + ( + crate::vtable::size_of_obj(fx, info.unwrap()), + crate::vtable::min_align_of_obj(fx, info.unwrap()), + ) + } + ty::Slice(_) | ty::Str => { + let unit = layout.field(fx, 0); + // The info in this case is the length of the str, so the size is that + // times the unit size. + ( + fx.bcx.ins().imul_imm(info.unwrap(), unit.size.bytes() as i64), + fx.bcx.ins().iconst(fx.pointer_type, unit.align.abi.bytes() as i64), + ) + } + ty::Foreign(_) => { + let trap_block = fx.bcx.create_block(); + let true_ = fx.bcx.ins().iconst(types::I8, 1); + let next_block = fx.bcx.create_block(); + fx.bcx.ins().brif(true_, trap_block, &[], next_block, &[]); + fx.bcx.seal_block(trap_block); + fx.bcx.seal_block(next_block); + fx.bcx.switch_to_block(trap_block); + + // `extern` type. We cannot compute the size, so panic. + let msg_str = with_no_visible_paths!({ + with_no_trimmed_paths!({ + format!("attempted to compute the size or alignment of extern type `{ty}`") + }) + }); + + codegen_panic_nounwind(fx, &msg_str, None); + + fx.bcx.switch_to_block(next_block); + + // This function does not return so we can now return whatever we want. + let size = fx.bcx.ins().iconst(fx.pointer_type, 42); + let align = fx.bcx.ins().iconst(fx.pointer_type, 42); + (size, align) + } + ty::Adt(..) | ty::Tuple(..) => { + // First get the size of all statically known fields. + // Don't use size_of because it also rounds up to alignment, which we + // want to avoid, as the unsized field's alignment could be smaller. + assert!(!layout.ty.is_simd()); + + let i = layout.fields.count() - 1; + let unsized_offset_unadjusted = layout.fields.offset(i).bytes(); + let unsized_offset_unadjusted = + fx.bcx.ins().iconst(fx.pointer_type, unsized_offset_unadjusted as i64); + let sized_align = layout.align.abi.bytes(); + let sized_align = fx.bcx.ins().iconst(fx.pointer_type, sized_align as i64); + + // Recurse to get the size of the dynamically sized field (must be + // the last field). + let field_layout = layout.field(fx, i); + let (unsized_size, mut unsized_align) = size_and_align_of(fx, field_layout, info); + + // # First compute the dynamic alignment + + // For packed types, we need to cap the alignment. + if let ty::Adt(def, _) = ty.kind() { + if let Some(packed) = def.repr().pack { + if packed.bytes() == 1 { + // We know this will be capped to 1. + unsized_align = fx.bcx.ins().iconst(fx.pointer_type, 1); + } else { + // We have to dynamically compute `min(unsized_align, packed)`. + let packed = fx.bcx.ins().iconst(fx.pointer_type, packed.bytes() as i64); + let cmp = fx.bcx.ins().icmp(IntCC::UnsignedLessThan, unsized_align, packed); + unsized_align = fx.bcx.ins().select(cmp, unsized_align, packed); + } + } + } + + // Choose max of two known alignments (combined value must + // be aligned according to more restrictive of the two). + let cmp = fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align); + let full_align = fx.bcx.ins().select(cmp, sized_align, unsized_align); + + // # Then compute the dynamic size + + // The full formula for the size would be: + // let unsized_offset_adjusted = unsized_offset_unadjusted.align_to(unsized_align); + // let full_size = (unsized_offset_adjusted + unsized_size).align_to(full_align); + // However, `unsized_size` is a multiple of `unsized_align`. + // Therefore, we can equivalently do the `align_to(unsized_align)` *after* adding `unsized_size`: + // let full_size = (unsized_offset_unadjusted + unsized_size).align_to(unsized_align).align_to(full_align); + // Furthermore, `align >= unsized_align`, and therefore we only need to do: + // let full_size = (unsized_offset_unadjusted + unsized_size).align_to(full_align); + + let full_size = fx.bcx.ins().iadd(unsized_offset_unadjusted, unsized_size); + + // Issue #27023: must add any necessary padding to `size` + // (to make it a multiple of `align`) before returning it. + // + // Namely, the returned size should be, in C notation: + // + // `size + ((size & (align-1)) ? align : 0)` + // + // emulated via the semi-standard fast bit trick: + // + // `(size + (align-1)) & -align` + let addend = fx.bcx.ins().iadd_imm(full_align, -1); + let add = fx.bcx.ins().iadd(full_size, addend); + let neg = fx.bcx.ins().ineg(full_align); + let full_size = fx.bcx.ins().band(add, neg); + + (full_size, full_align) + } + _ => bug!("size_and_align_of_dst: {ty} not supported"), + } +} diff --git a/compiler/rustc_codegen_cranelift/src/unwind_module.rs b/compiler/rustc_codegen_cranelift/src/unwind_module.rs new file mode 100644 index 00000000000..b950aaa29ce --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/unwind_module.rs @@ -0,0 +1,115 @@ +use cranelift_codegen::control::ControlPlane; +use cranelift_codegen::ir::{Function, Signature}; +use cranelift_codegen::isa::{TargetFrontendConfig, TargetIsa}; +use cranelift_codegen::{Context, FinalizedMachReloc}; +use cranelift_module::{ + DataDescription, DataId, FuncId, FuncOrDataId, Linkage, Module, ModuleDeclarations, + ModuleResult, +}; +use cranelift_object::{ObjectModule, ObjectProduct}; + +use crate::UnwindContext; + +/// A wrapper around a [Module] which adds any defined function to the [UnwindContext]. +pub(crate) struct UnwindModule<T> { + pub(crate) module: T, + unwind_context: UnwindContext, +} + +impl<T: Module> UnwindModule<T> { + pub(crate) fn new(module: T, pic_eh_frame: bool) -> Self { + let unwind_context = UnwindContext::new(module.isa(), pic_eh_frame); + UnwindModule { module, unwind_context } + } +} + +impl UnwindModule<ObjectModule> { + pub(crate) fn finish(self) -> ObjectProduct { + let mut product = self.module.finish(); + self.unwind_context.emit(&mut product); + product + } +} + +#[cfg(feature = "jit")] +impl UnwindModule<cranelift_jit::JITModule> { + pub(crate) fn finalize_definitions(&mut self) { + self.module.finalize_definitions().unwrap(); + let prev_unwind_context = std::mem::replace( + &mut self.unwind_context, + UnwindContext::new(self.module.isa(), false), + ); + unsafe { prev_unwind_context.register_jit(&self.module) }; + } +} + +impl<T: Module> Module for UnwindModule<T> { + fn isa(&self) -> &dyn TargetIsa { + self.module.isa() + } + + fn declarations(&self) -> &ModuleDeclarations { + self.module.declarations() + } + + fn get_name(&self, name: &str) -> Option<FuncOrDataId> { + self.module.get_name(name) + } + + fn target_config(&self) -> TargetFrontendConfig { + self.module.target_config() + } + + fn declare_function( + &mut self, + name: &str, + linkage: Linkage, + signature: &Signature, + ) -> ModuleResult<FuncId> { + self.module.declare_function(name, linkage, signature) + } + + fn declare_anonymous_function(&mut self, signature: &Signature) -> ModuleResult<FuncId> { + self.module.declare_anonymous_function(signature) + } + + fn declare_data( + &mut self, + name: &str, + linkage: Linkage, + writable: bool, + tls: bool, + ) -> ModuleResult<DataId> { + self.module.declare_data(name, linkage, writable, tls) + } + + fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult<DataId> { + self.module.declare_anonymous_data(writable, tls) + } + + fn define_function_with_control_plane( + &mut self, + func: FuncId, + ctx: &mut Context, + ctrl_plane: &mut ControlPlane, + ) -> ModuleResult<()> { + self.module.define_function_with_control_plane(func, ctx, ctrl_plane)?; + self.unwind_context.add_function(func, ctx, self.module.isa()); + Ok(()) + } + + fn define_function_bytes( + &mut self, + _func_id: FuncId, + _func: &Function, + _alignment: u64, + _bytes: &[u8], + _relocs: &[FinalizedMachReloc], + ) -> ModuleResult<()> { + unimplemented!() + } + + fn define_data(&mut self, data_id: DataId, data: &DataDescription) -> ModuleResult<()> { + self.module.define_data(data_id, data) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs new file mode 100644 index 00000000000..8eb2095e523 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -0,0 +1,1015 @@ +//! Definition of [`CValue`] and [`CPlace`] + +use cranelift_codegen::entity::EntityRef; +use cranelift_codegen::ir::immediates::Offset32; +use cranelift_frontend::Variable; +use rustc_middle::ty::FnSig; + +use crate::prelude::*; + +fn codegen_field<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + base: Pointer, + extra: Option<Value>, + layout: TyAndLayout<'tcx>, + field: FieldIdx, +) -> (Pointer, TyAndLayout<'tcx>) { + let field_offset = layout.fields.offset(field.index()); + let field_layout = layout.field(&*fx, field.index()); + + let simple = |fx: &mut FunctionCx<'_, '_, '_>| { + (base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), field_layout) + }; + + if field_layout.is_sized() { + return simple(fx); + } + match field_layout.ty.kind() { + ty::Slice(..) | ty::Str => simple(fx), + _ => { + let unaligned_offset = field_offset.bytes(); + + // Get the alignment of the field + let (_, mut unsized_align) = crate::unsize::size_and_align_of(fx, field_layout, extra); + + // For packed types, we need to cap alignment. + if let ty::Adt(def, _) = layout.ty.kind() { + if let Some(packed) = def.repr().pack { + let packed = fx.bcx.ins().iconst(fx.pointer_type, packed.bytes() as i64); + let cmp = fx.bcx.ins().icmp(IntCC::UnsignedLessThan, unsized_align, packed); + unsized_align = fx.bcx.ins().select(cmp, unsized_align, packed); + } + } + + // Bump the unaligned offset up to the appropriate alignment + let one = fx.bcx.ins().iconst(fx.pointer_type, 1); + let align_sub_1 = fx.bcx.ins().isub(unsized_align, one); + let and_lhs = fx.bcx.ins().iadd_imm(align_sub_1, unaligned_offset as i64); + let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); + let and_rhs = fx.bcx.ins().isub(zero, unsized_align); + let offset = fx.bcx.ins().band(and_lhs, and_rhs); + + (base.offset_value(fx, offset), field_layout) + } + } +} + +fn scalar_pair_calculate_b_offset(tcx: TyCtxt<'_>, a_scalar: Scalar, b_scalar: Scalar) -> Offset32 { + let b_offset = a_scalar.size(&tcx).align_to(b_scalar.align(&tcx).abi); + Offset32::new(b_offset.bytes().try_into().unwrap()) +} + +/// A read-only value +#[derive(Debug, Copy, Clone)] +pub(crate) struct CValue<'tcx>(CValueInner, TyAndLayout<'tcx>); + +#[derive(Debug, Copy, Clone)] +enum CValueInner { + ByRef(Pointer, Option<Value>), + ByVal(Value), + ByValPair(Value, Value), +} + +impl<'tcx> CValue<'tcx> { + pub(crate) fn by_ref(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CValue<'tcx> { + CValue(CValueInner::ByRef(ptr, None), layout) + } + + pub(crate) fn by_ref_unsized( + ptr: Pointer, + meta: Value, + layout: TyAndLayout<'tcx>, + ) -> CValue<'tcx> { + CValue(CValueInner::ByRef(ptr, Some(meta)), layout) + } + + pub(crate) fn by_val(value: Value, layout: TyAndLayout<'tcx>) -> CValue<'tcx> { + CValue(CValueInner::ByVal(value), layout) + } + + pub(crate) fn by_val_pair( + value: Value, + extra: Value, + layout: TyAndLayout<'tcx>, + ) -> CValue<'tcx> { + CValue(CValueInner::ByValPair(value, extra), layout) + } + + /// Create an instance of a ZST + /// + /// The is represented by a dangling pointer of suitable alignment. + pub(crate) fn zst(layout: TyAndLayout<'tcx>) -> CValue<'tcx> { + assert!(layout.is_zst()); + CValue::by_ref(crate::Pointer::dangling(layout.align.pref), layout) + } + + pub(crate) fn layout(&self) -> TyAndLayout<'tcx> { + self.1 + } + + // FIXME remove + pub(crate) fn force_stack(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Pointer, Option<Value>) { + let layout = self.1; + match self.0 { + CValueInner::ByRef(ptr, meta) => (ptr, meta), + CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => { + let cplace = CPlace::new_stack_slot(fx, layout); + cplace.write_cvalue(fx, self); + (cplace.to_ptr(), None) + } + } + } + + // FIXME remove + /// Forces the data value of a dyn* value to the stack and returns a pointer to it as well as the + /// vtable pointer. + pub(crate) fn dyn_star_force_data_on_stack( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + ) -> (Value, Value) { + assert!(self.1.ty.is_dyn_star()); + + match self.0 { + CValueInner::ByRef(ptr, None) => { + let (a_scalar, b_scalar) = match self.1.abi { + Abi::ScalarPair(a, b) => (a, b), + _ => unreachable!("dyn_star_force_data_on_stack({:?})", self), + }; + let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar); + let mut flags = MemFlags::new(); + flags.set_notrap(); + let vtable = ptr.offset(fx, b_offset).load(fx, clif_ty2, flags); + (ptr.get_addr(fx), vtable) + } + CValueInner::ByValPair(data, vtable) => { + let data_ptr = fx.create_stack_slot( + u32::try_from(fx.target_config.pointer_type().bytes()).unwrap(), + u32::try_from(fx.target_config.pointer_type().bytes()).unwrap(), + ); + data_ptr.store(fx, data, MemFlags::trusted()); + + (data_ptr.get_addr(fx), vtable) + } + CValueInner::ByRef(_, Some(_)) | CValueInner::ByVal(_) => { + unreachable!("dyn_star_force_data_on_stack({:?})", self) + } + } + } + + pub(crate) fn try_to_ptr(self) -> Option<(Pointer, Option<Value>)> { + match self.0 { + CValueInner::ByRef(ptr, meta) => Some((ptr, meta)), + CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => None, + } + } + + /// Load a value with layout.abi of scalar + #[track_caller] + pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value { + let layout = self.1; + match self.0 { + CValueInner::ByRef(ptr, None) => { + let clif_ty = match layout.abi { + Abi::Scalar(scalar) => scalar_to_clif_type(fx.tcx, scalar), + Abi::Vector { element, count } => scalar_to_clif_type(fx.tcx, element) + .by(u32::try_from(count).unwrap()) + .unwrap(), + _ => unreachable!("{:?}", layout.ty), + }; + let mut flags = MemFlags::new(); + flags.set_notrap(); + ptr.load(fx, clif_ty, flags) + } + CValueInner::ByVal(value) => value, + CValueInner::ByRef(_, Some(_)) => bug!("load_scalar for unsized value not allowed"), + CValueInner::ByValPair(_, _) => bug!("Please use load_scalar_pair for ByValPair"), + } + } + + /// Load a value pair with layout.abi of scalar pair + #[track_caller] + pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) { + let layout = self.1; + match self.0 { + CValueInner::ByRef(ptr, None) => { + let (a_scalar, b_scalar) = match layout.abi { + Abi::ScalarPair(a, b) => (a, b), + _ => unreachable!("load_scalar_pair({:?})", self), + }; + let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + let clif_ty1 = scalar_to_clif_type(fx.tcx, a_scalar); + let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar); + let mut flags = MemFlags::new(); + flags.set_notrap(); + let val1 = ptr.load(fx, clif_ty1, flags); + let val2 = ptr.offset(fx, b_offset).load(fx, clif_ty2, flags); + (val1, val2) + } + CValueInner::ByRef(_, Some(_)) => { + bug!("load_scalar_pair for unsized value not allowed") + } + CValueInner::ByVal(_) => bug!("Please use load_scalar for ByVal"), + CValueInner::ByValPair(val1, val2) => (val1, val2), + } + } + + pub(crate) fn value_field( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + field: FieldIdx, + ) -> CValue<'tcx> { + let layout = self.1; + match self.0 { + CValueInner::ByVal(_) => unreachable!(), + CValueInner::ByValPair(val1, val2) => match layout.abi { + Abi::ScalarPair(_, _) => { + let val = match field.as_u32() { + 0 => val1, + 1 => val2, + _ => bug!("field should be 0 or 1"), + }; + let field_layout = layout.field(&*fx, usize::from(field)); + CValue::by_val(val, field_layout) + } + _ => unreachable!("value_field for ByValPair with abi {:?}", layout.abi), + }, + CValueInner::ByRef(ptr, None) => { + let (field_ptr, field_layout) = codegen_field(fx, ptr, None, layout, field); + CValue::by_ref(field_ptr, field_layout) + } + CValueInner::ByRef(_, Some(_)) => todo!(), + } + } + + /// Like [`CValue::value_field`] except handling ADTs containing a single array field in a way + /// such that you can access individual lanes. + pub(crate) fn value_lane( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + lane_idx: u64, + ) -> CValue<'tcx> { + let layout = self.1; + assert!(layout.ty.is_simd()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + assert!(lane_idx < lane_count); + + match self.0 { + CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => unreachable!(), + CValueInner::ByRef(ptr, None) => { + let field_offset = lane_layout.size * lane_idx; + let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()); + CValue::by_ref(field_ptr, lane_layout) + } + CValueInner::ByRef(_, Some(_)) => unreachable!(), + } + } + + /// Like [`CValue::value_field`] except using the passed type as lane type instead of the one + /// specified by the vector type. + pub(crate) fn value_typed_lane( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + lane_ty: Ty<'tcx>, + lane_idx: u64, + ) -> CValue<'tcx> { + let layout = self.1; + assert!(layout.ty.is_simd()); + let (orig_lane_count, orig_lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + assert!( + (lane_idx + 1) * lane_layout.size <= orig_lane_count * fx.layout_of(orig_lane_ty).size + ); + + match self.0 { + CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => unreachable!(), + CValueInner::ByRef(ptr, None) => { + let field_offset = lane_layout.size * lane_idx; + let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()); + CValue::by_ref(field_ptr, lane_layout) + } + CValueInner::ByRef(_, Some(_)) => unreachable!(), + } + } + + /// Like [`CValue::value_lane`] except allowing a dynamically calculated lane index. + pub(crate) fn value_lane_dyn( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + lane_idx: Value, + ) -> CValue<'tcx> { + let layout = self.1; + assert!(layout.ty.is_simd()); + let (_lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + match self.0 { + CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => unreachable!(), + CValueInner::ByRef(ptr, None) => { + let field_offset = fx.bcx.ins().imul_imm(lane_idx, lane_layout.size.bytes() as i64); + let field_ptr = ptr.offset_value(fx, field_offset); + CValue::by_ref(field_ptr, lane_layout) + } + CValueInner::ByRef(_, Some(_)) => unreachable!(), + } + } + + /// If `ty` is signed, `const_val` must already be sign extended. + pub(crate) fn const_val( + fx: &mut FunctionCx<'_, '_, 'tcx>, + layout: TyAndLayout<'tcx>, + const_val: ty::ScalarInt, + ) -> CValue<'tcx> { + assert_eq!(const_val.size(), layout.size, "{:#?}: {:?}", const_val, layout); + use cranelift_codegen::ir::immediates::{Ieee32, Ieee64}; + + let clif_ty = fx.clif_type(layout.ty).unwrap(); + + let val = match layout.ty.kind() { + ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { + let const_val = const_val.to_bits(layout.size); + let lsb = fx.bcx.ins().iconst(types::I64, const_val as u64 as i64); + let msb = fx.bcx.ins().iconst(types::I64, (const_val >> 64) as u64 as i64); + fx.bcx.ins().iconcat(lsb, msb) + } + ty::Bool + | ty::Char + | ty::Uint(_) + | ty::Int(_) + | ty::Ref(..) + | ty::RawPtr(..) + | ty::FnPtr(..) => { + let raw_val = const_val.size().truncate(const_val.to_bits(layout.size)); + fx.bcx.ins().iconst(clif_ty, raw_val as i64) + } + ty::Float(FloatTy::F32) => { + fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap())) + } + ty::Float(FloatTy::F64) => { + fx.bcx.ins().f64const(Ieee64::with_bits(u64::try_from(const_val).unwrap())) + } + _ => panic!( + "CValue::const_val for non bool/char/float/integer/pointer type {:?} is not allowed", + layout.ty + ), + }; + + CValue::by_val(val, layout) + } + + pub(crate) fn cast_pointer_to(self, layout: TyAndLayout<'tcx>) -> Self { + assert!(matches!(self.layout().ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..))); + assert!(matches!(layout.ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..))); + assert_eq!(self.layout().abi, layout.abi); + CValue(self.0, layout) + } +} + +/// A place where you can write a value to or read a value from +#[derive(Debug, Copy, Clone)] +pub(crate) struct CPlace<'tcx> { + inner: CPlaceInner, + layout: TyAndLayout<'tcx>, +} + +#[derive(Debug, Copy, Clone)] +enum CPlaceInner { + Var(Local, Variable), + VarPair(Local, Variable, Variable), + Addr(Pointer, Option<Value>), +} + +impl<'tcx> CPlace<'tcx> { + pub(crate) fn layout(&self) -> TyAndLayout<'tcx> { + self.layout + } + + pub(crate) fn new_stack_slot( + fx: &mut FunctionCx<'_, '_, 'tcx>, + layout: TyAndLayout<'tcx>, + ) -> CPlace<'tcx> { + assert!(layout.is_sized()); + if layout.size.bytes() == 0 { + return CPlace { + inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None), + layout, + }; + } + + if layout.size.bytes() >= u64::from(u32::MAX - 16) { + fx.tcx + .dcx() + .fatal(format!("values of type {} are too big to store on the stack", layout.ty)); + } + + let stack_slot = fx.create_stack_slot( + u32::try_from(layout.size.bytes()).unwrap(), + u32::try_from(layout.align.pref.bytes()).unwrap(), + ); + CPlace { inner: CPlaceInner::Addr(stack_slot, None), layout } + } + + pub(crate) fn new_var( + fx: &mut FunctionCx<'_, '_, 'tcx>, + local: Local, + layout: TyAndLayout<'tcx>, + ) -> CPlace<'tcx> { + let var = Variable::from_u32(fx.next_ssa_var); + fx.next_ssa_var += 1; + fx.bcx.declare_var(var, fx.clif_type(layout.ty).unwrap()); + CPlace { inner: CPlaceInner::Var(local, var), layout } + } + + pub(crate) fn new_var_pair( + fx: &mut FunctionCx<'_, '_, 'tcx>, + local: Local, + layout: TyAndLayout<'tcx>, + ) -> CPlace<'tcx> { + let var1 = Variable::from_u32(fx.next_ssa_var); + fx.next_ssa_var += 1; + let var2 = Variable::from_u32(fx.next_ssa_var); + fx.next_ssa_var += 1; + + let (ty1, ty2) = fx.clif_pair_type(layout.ty).unwrap(); + fx.bcx.declare_var(var1, ty1); + fx.bcx.declare_var(var2, ty2); + CPlace { inner: CPlaceInner::VarPair(local, var1, var2), layout } + } + + pub(crate) fn for_ptr(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CPlace<'tcx> { + CPlace { inner: CPlaceInner::Addr(ptr, None), layout } + } + + pub(crate) fn for_ptr_with_extra( + ptr: Pointer, + extra: Value, + layout: TyAndLayout<'tcx>, + ) -> CPlace<'tcx> { + CPlace { inner: CPlaceInner::Addr(ptr, Some(extra)), layout } + } + + pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CValue<'tcx> { + let layout = self.layout(); + match self.inner { + CPlaceInner::Var(_local, var) => { + let val = fx.bcx.use_var(var); + //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index())); + CValue::by_val(val, layout) + } + CPlaceInner::VarPair(_local, var1, var2) => { + let val1 = fx.bcx.use_var(var1); + //fx.bcx.set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index())); + let val2 = fx.bcx.use_var(var2); + //fx.bcx.set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index())); + CValue::by_val_pair(val1, val2, layout) + } + CPlaceInner::Addr(ptr, extra) => { + if let Some(extra) = extra { + CValue::by_ref_unsized(ptr, extra, layout) + } else { + CValue::by_ref(ptr, layout) + } + } + } + } + + pub(crate) fn debug_comment(self) -> (&'static str, String) { + match self.inner { + CPlaceInner::Var(_local, var) => ("ssa", format!("var={}", var.index())), + CPlaceInner::VarPair(_local, var1, var2) => { + ("ssa", format!("var=({}, {})", var1.index(), var2.index())) + } + CPlaceInner::Addr(ptr, meta) => { + let meta = + if let Some(meta) = meta { format!(",meta={}", meta) } else { String::new() }; + match ptr.debug_base_and_offset() { + (crate::pointer::PointerBase::Addr(addr), offset) => { + ("reuse", format!("storage={}{}{}", addr, offset, meta)) + } + (crate::pointer::PointerBase::Stack(stack_slot), offset) => { + ("stack", format!("storage={}{}{}", stack_slot, offset, meta)) + } + (crate::pointer::PointerBase::Dangling(align), offset) => { + ("zst", format!("align={},offset={}", align.bytes(), offset)) + } + } + } + } + } + + #[track_caller] + pub(crate) fn to_ptr(self) -> Pointer { + match self.inner { + CPlaceInner::Addr(ptr, None) => ptr, + CPlaceInner::Addr(_, Some(_)) => bug!("Expected sized cplace, found {:?}", self), + CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => { + bug!("Expected CPlace::Addr, found {:?}", self) + } + } + } + + #[track_caller] + pub(crate) fn to_ptr_unsized(self) -> (Pointer, Value) { + match self.inner { + CPlaceInner::Addr(ptr, Some(extra)) => (ptr, extra), + CPlaceInner::Addr(_, None) | CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => { + bug!("Expected unsized cplace, found {:?}", self) + } + } + } + + pub(crate) fn try_to_ptr(self) -> Option<Pointer> { + match self.inner { + CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => None, + CPlaceInner::Addr(ptr, None) => Some(ptr), + CPlaceInner::Addr(_, Some(_)) => bug!("Expected sized cplace, found {:?}", self), + } + } + + pub(crate) fn write_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>, from: CValue<'tcx>) { + assert_assignable(fx, from.layout().ty, self.layout().ty, 16); + + self.write_cvalue_maybe_transmute(fx, from, "write_cvalue"); + } + + pub(crate) fn write_cvalue_transmute( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + from: CValue<'tcx>, + ) { + self.write_cvalue_maybe_transmute(fx, from, "write_cvalue_transmute"); + } + + fn write_cvalue_maybe_transmute( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + from: CValue<'tcx>, + method: &'static str, + ) { + fn transmute_scalar<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + var: Variable, + data: Value, + dst_ty: Type, + ) { + let src_ty = fx.bcx.func.dfg.value_type(data); + assert_eq!( + src_ty.bytes(), + dst_ty.bytes(), + "write_cvalue_transmute: {:?} -> {:?}", + src_ty, + dst_ty, + ); + let data = match (src_ty, dst_ty) { + (_, _) if src_ty == dst_ty => data, + + // This is a `write_cvalue_transmute`. + (types::I32, types::F32) + | (types::F32, types::I32) + | (types::I64, types::F64) + | (types::F64, types::I64) => codegen_bitcast(fx, dst_ty, data), + _ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data), + _ if src_ty.is_vector() || dst_ty.is_vector() => { + // FIXME(bytecodealliance/wasmtime#6104) do something more efficient for transmutes between vectors and integers. + let ptr = fx.create_stack_slot(src_ty.bytes(), src_ty.bytes()); + ptr.store(fx, data, MemFlags::trusted()); + ptr.load(fx, dst_ty, MemFlags::trusted()) + } + + // `CValue`s should never contain SSA-only types, so if you ended + // up here having seen an error like `B1 -> I8`, then before + // calling `write_cvalue` you need to add a `bint` instruction. + _ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty), + }; + //fx.bcx.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index())); + fx.bcx.def_var(var, data); + } + + assert_eq!(self.layout().size, from.layout().size); + + if fx.clif_comments.enabled() { + use cranelift_codegen::cursor::{Cursor, CursorPosition}; + let cur_block = match fx.bcx.cursor().position() { + CursorPosition::After(block) => block, + _ => unreachable!(), + }; + fx.add_comment( + fx.bcx.func.layout.last_inst(cur_block).unwrap(), + format!( + "{}: {:?}: {:?} <- {:?}: {:?}", + method, + self.inner, + self.layout().ty, + from.0, + from.layout().ty + ), + ); + } + + let dst_layout = self.layout(); + match self.inner { + CPlaceInner::Var(_local, var) => { + let data = match from.1.abi { + Abi::Scalar(_) => CValue(from.0, dst_layout).load_scalar(fx), + _ => { + let (ptr, meta) = from.force_stack(fx); + assert!(meta.is_none()); + CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar(fx) + } + }; + let dst_ty = fx.clif_type(self.layout().ty).unwrap(); + transmute_scalar(fx, var, data, dst_ty); + } + CPlaceInner::VarPair(_local, var1, var2) => { + let (data1, data2) = match from.1.abi { + Abi::ScalarPair(_, _) => CValue(from.0, dst_layout).load_scalar_pair(fx), + _ => { + let (ptr, meta) = from.force_stack(fx); + assert!(meta.is_none()); + CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx) + } + }; + let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap(); + transmute_scalar(fx, var1, data1, dst_ty1); + transmute_scalar(fx, var2, data2, dst_ty2); + } + CPlaceInner::Addr(_, Some(_)) => bug!("Can't write value to unsized place {:?}", self), + CPlaceInner::Addr(to_ptr, None) => { + if dst_layout.size == Size::ZERO || dst_layout.abi == Abi::Uninhabited { + return; + } + + let mut flags = MemFlags::new(); + flags.set_notrap(); + + match from.0 { + CValueInner::ByVal(val) => { + to_ptr.store(fx, val, flags); + } + CValueInner::ByValPair(val1, val2) => match from.layout().abi { + Abi::ScalarPair(a_scalar, b_scalar) => { + let b_offset = + scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + to_ptr.store(fx, val1, flags); + to_ptr.offset(fx, b_offset).store(fx, val2, flags); + } + _ => bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi), + }, + CValueInner::ByRef(from_ptr, None) => { + match from.layout().abi { + Abi::Scalar(_) => { + let val = from.load_scalar(fx); + to_ptr.store(fx, val, flags); + return; + } + Abi::ScalarPair(a_scalar, b_scalar) => { + let b_offset = + scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + let (val1, val2) = from.load_scalar_pair(fx); + to_ptr.store(fx, val1, flags); + to_ptr.offset(fx, b_offset).store(fx, val2, flags); + return; + } + _ => {} + } + + let from_addr = from_ptr.get_addr(fx); + let to_addr = to_ptr.get_addr(fx); + let src_layout = from.1; + let size = dst_layout.size.bytes(); + // `emit_small_memory_copy` uses `u8` for alignments, just use the maximum + // alignment that fits in a `u8` if the actual alignment is larger. + let src_align = src_layout.align.abi.bytes().try_into().unwrap_or(128); + let dst_align = dst_layout.align.abi.bytes().try_into().unwrap_or(128); + fx.bcx.emit_small_memory_copy( + fx.target_config, + to_addr, + from_addr, + size, + dst_align, + src_align, + true, + flags, + ); + } + CValueInner::ByRef(_, Some(_)) => todo!(), + } + } + } + } + + /// Used for `ProjectionElem::Subtype`, `ty` has to be monomorphized before + /// passed on. + pub(crate) fn place_transmute_type( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + ty: Ty<'tcx>, + ) -> CPlace<'tcx> { + CPlace { inner: self.inner, layout: fx.layout_of(ty) } + } + + pub(crate) fn place_field( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + field: FieldIdx, + ) -> CPlace<'tcx> { + let layout = self.layout(); + + match self.inner { + CPlaceInner::VarPair(local, var1, var2) => { + let layout = layout.field(&*fx, field.index()); + + match field.as_u32() { + 0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout }, + 1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout }, + _ => unreachable!("field should be 0 or 1"), + } + } + _ => {} + } + + let (base, extra) = match self.inner { + CPlaceInner::Addr(ptr, extra) => (ptr, extra), + CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => { + bug!("Expected CPlace::Addr, found {:?}", self) + } + }; + + let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field); + if has_ptr_meta(fx.tcx, field_layout.ty) { + CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout) + } else { + CPlace::for_ptr(field_ptr, field_layout) + } + } + + /// Like [`CPlace::place_field`] except handling ADTs containing a single array field in a way + /// such that you can access individual lanes. + pub(crate) fn place_lane( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + lane_idx: u64, + ) -> CPlace<'tcx> { + let layout = self.layout(); + assert!(layout.ty.is_simd()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + assert!(lane_idx < lane_count); + + match self.inner { + CPlaceInner::Var(_, _) => unreachable!(), + CPlaceInner::VarPair(_, _, _) => unreachable!(), + CPlaceInner::Addr(ptr, None) => { + let field_offset = lane_layout.size * lane_idx; + let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()); + CPlace::for_ptr(field_ptr, lane_layout) + } + CPlaceInner::Addr(_, Some(_)) => unreachable!(), + } + } + + /// Like [`CPlace::place_field`] except using the passed type as lane type instead of the one + /// specified by the vector type. + pub(crate) fn place_typed_lane( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + lane_ty: Ty<'tcx>, + lane_idx: u64, + ) -> CPlace<'tcx> { + let layout = self.layout(); + assert!(layout.ty.is_simd()); + let (orig_lane_count, orig_lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + assert!( + (lane_idx + 1) * lane_layout.size <= orig_lane_count * fx.layout_of(orig_lane_ty).size + ); + + match self.inner { + CPlaceInner::Var(_, _) => unreachable!(), + CPlaceInner::VarPair(_, _, _) => unreachable!(), + CPlaceInner::Addr(ptr, None) => { + let field_offset = lane_layout.size * lane_idx; + let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()); + CPlace::for_ptr(field_ptr, lane_layout) + } + CPlaceInner::Addr(_, Some(_)) => unreachable!(), + } + } + + pub(crate) fn place_index( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + index: Value, + ) -> CPlace<'tcx> { + let (elem_layout, ptr) = match self.layout().ty.kind() { + ty::Array(elem_ty, _) => { + let elem_layout = fx.layout_of(*elem_ty); + match self.inner { + CPlaceInner::Addr(addr, None) => (elem_layout, addr), + CPlaceInner::Var(_, _) + | CPlaceInner::Addr(_, Some(_)) + | CPlaceInner::VarPair(_, _, _) => bug!("Can't index into {self:?}"), + } + } + ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_unsized().0), + _ => bug!("place_index({:?})", self.layout().ty), + }; + + let offset = fx.bcx.ins().imul_imm(index, elem_layout.size.bytes() as i64); + + CPlace::for_ptr(ptr.offset_value(fx, offset), elem_layout) + } + + pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CPlace<'tcx> { + let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap()); + if has_ptr_meta(fx.tcx, inner_layout.ty) { + let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx); + CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout) + } else { + CPlace::for_ptr(Pointer::new(self.to_cvalue(fx).load_scalar(fx)), inner_layout) + } + } + + pub(crate) fn place_ref( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + layout: TyAndLayout<'tcx>, + ) -> CValue<'tcx> { + if has_ptr_meta(fx.tcx, self.layout().ty) { + let (ptr, extra) = self.to_ptr_unsized(); + CValue::by_val_pair(ptr.get_addr(fx), extra, layout) + } else { + CValue::by_val(self.to_ptr().get_addr(fx), layout) + } + } + + pub(crate) fn downcast_variant( + self, + fx: &FunctionCx<'_, '_, 'tcx>, + variant: VariantIdx, + ) -> Self { + assert!(self.layout().is_sized()); + let layout = self.layout().for_variant(fx, variant); + CPlace { inner: self.inner, layout } + } +} + +#[track_caller] +pub(crate) fn assert_assignable<'tcx>( + fx: &FunctionCx<'_, '_, 'tcx>, + from_ty: Ty<'tcx>, + to_ty: Ty<'tcx>, + limit: usize, +) { + if limit == 0 { + // assert_assignable exists solely to catch bugs in cg_clif. it isn't necessary for + // soundness. don't attempt to check deep types to avoid exponential behavior in certain + // cases. + return; + } + match (from_ty.kind(), to_ty.kind()) { + (ty::Ref(_, a, _), ty::Ref(_, b, _)) | (ty::RawPtr(a, _), ty::RawPtr(b, _)) => { + assert_assignable(fx, *a, *b, limit - 1); + } + (ty::Ref(_, a, _), ty::RawPtr(b, _)) | (ty::RawPtr(a, _), ty::Ref(_, b, _)) => { + assert_assignable(fx, *a, *b, limit - 1); + } + (ty::FnPtr(_), ty::FnPtr(_)) => { + let from_sig = fx.tcx.normalize_erasing_late_bound_regions( + ParamEnv::reveal_all(), + from_ty.fn_sig(fx.tcx), + ); + let FnSig { + inputs_and_output: types_from, + c_variadic: c_variadic_from, + safety: unsafety_from, + abi: abi_from, + } = from_sig; + let to_sig = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx)); + let FnSig { + inputs_and_output: types_to, + c_variadic: c_variadic_to, + safety: unsafety_to, + abi: abi_to, + } = to_sig; + let mut types_from = types_from.iter(); + let mut types_to = types_to.iter(); + loop { + match (types_from.next(), types_to.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => break, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + assert_eq!( + c_variadic_from, c_variadic_to, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); + assert_eq!( + unsafety_from, unsafety_to, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); + assert_eq!( + abi_from, abi_to, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); + // fn(&T) -> for<'l> fn(&'l T) is allowed + } + (&ty::Dynamic(from_traits, _, _from_kind), &ty::Dynamic(to_traits, _, _to_kind)) => { + // FIXME(dyn-star): Do the right thing with DynKinds + for (from, to) in from_traits.iter().zip(to_traits) { + let from = + fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from); + let to = fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to); + assert_eq!( + from, to, + "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}", + from_traits, to_traits, fx, + ); + } + // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed + } + (&ty::Tuple(types_a), &ty::Tuple(types_b)) => { + let mut types_a = types_a.iter(); + let mut types_b = types_b.iter(); + loop { + match (types_a.next(), types_b.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => return, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + } + (&ty::Adt(adt_def_a, args_a), &ty::Adt(adt_def_b, args_b)) + if adt_def_a.did() == adt_def_b.did() => + { + let mut types_a = args_a.types(); + let mut types_b = args_b.types(); + loop { + match (types_a.next(), types_b.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => return, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + } + (ty::Array(a, _), ty::Array(b, _)) => assert_assignable(fx, *a, *b, limit - 1), + (&ty::Closure(def_id_a, args_a), &ty::Closure(def_id_b, args_b)) + if def_id_a == def_id_b => + { + let mut types_a = args_a.types(); + let mut types_b = args_b.types(); + loop { + match (types_a.next(), types_b.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => return, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + } + (&ty::Coroutine(def_id_a, args_a), &ty::Coroutine(def_id_b, args_b)) + if def_id_a == def_id_b => + { + let mut types_a = args_a.types(); + let mut types_b = args_b.types(); + loop { + match (types_a.next(), types_b.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => return, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + } + (&ty::CoroutineWitness(def_id_a, args_a), &ty::CoroutineWitness(def_id_b, args_b)) + if def_id_a == def_id_b => + { + let mut types_a = args_a.types(); + let mut types_b = args_b.types(); + loop { + match (types_a.next(), types_b.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => return, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + } + (ty::Param(_), _) | (_, ty::Param(_)) if fx.tcx.sess.opts.unstable_opts.polymorphize => { + // No way to check if it is correct or not with polymorphization enabled + } + _ => { + assert_eq!( + from_ty, + to_ty, + "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}", + from_ty.kind(), + to_ty.kind(), + fx, + ); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs new file mode 100644 index 00000000000..14c607ccad7 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/vtable.rs @@ -0,0 +1,101 @@ +//! Codegen vtables and vtable accesses. +//! +//! See `rustc_codegen_ssa/src/meth.rs` for reference. + +use crate::constant::data_id_for_vtable; +use crate::prelude::*; + +pub(crate) fn vtable_memflags() -> MemFlags { + let mut flags = MemFlags::trusted(); // A vtable access is always aligned and will never trap. + flags.set_readonly(); // A vtable is always read-only. + flags +} + +pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value { + let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize; + fx.bcx.ins().load( + fx.pointer_type, + vtable_memflags(), + vtable, + (ty::COMMON_VTABLE_ENTRIES_DROPINPLACE * usize_size) as i32, + ) +} + +pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value { + let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize; + fx.bcx.ins().load( + fx.pointer_type, + vtable_memflags(), + vtable, + (ty::COMMON_VTABLE_ENTRIES_SIZE * usize_size) as i32, + ) +} + +pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value { + let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize; + fx.bcx.ins().load( + fx.pointer_type, + vtable_memflags(), + vtable, + (ty::COMMON_VTABLE_ENTRIES_ALIGN * usize_size) as i32, + ) +} + +pub(crate) fn get_ptr_and_method_ref<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + mut arg: CValue<'tcx>, + idx: usize, +) -> (Pointer, Value) { + let (ptr, vtable) = 'block: { + if let Abi::Scalar(_) = arg.layout().abi { + while !arg.layout().ty.is_unsafe_ptr() && !arg.layout().ty.is_ref() { + let (idx, _) = arg + .layout() + .non_1zst_field(fx) + .expect("not exactly one non-1-ZST field in a `DispatchFromDyn` type"); + arg = arg.value_field(fx, FieldIdx::new(idx)); + } + } + + if let ty::Ref(_, ty, _) = arg.layout().ty.kind() { + if ty.is_dyn_star() { + let inner_layout = fx.layout_of(arg.layout().ty.builtin_deref(true).unwrap()); + let dyn_star = CPlace::for_ptr(Pointer::new(arg.load_scalar(fx)), inner_layout); + let ptr = dyn_star.place_field(fx, FieldIdx::ZERO).to_ptr(); + let vtable = + dyn_star.place_field(fx, FieldIdx::new(1)).to_cvalue(fx).load_scalar(fx); + break 'block (ptr, vtable); + } + } + + if let Abi::ScalarPair(_, _) = arg.layout().abi { + let (ptr, vtable) = arg.load_scalar_pair(fx); + (Pointer::new(ptr), vtable) + } else { + let (ptr, vtable) = arg.try_to_ptr().unwrap(); + (ptr, vtable.unwrap()) + } + }; + + let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes(); + let func_ref = fx.bcx.ins().load( + fx.pointer_type, + vtable_memflags(), + vtable, + (idx * usize_size as usize) as i32, + ); + (ptr, func_ref) +} + +pub(crate) fn get_vtable<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + ty: Ty<'tcx>, + trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, +) -> Value { + let data_id = data_id_for_vtable(fx.tcx, &mut fx.constants_cx, fx.module, ty, trait_ref); + let local_data_id = fx.module.declare_data_in_func(data_id, fx.bcx.func); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, "vtable"); + } + fx.bcx.ins().global_value(fx.pointer_type, local_data_id) +} diff --git a/compiler/rustc_codegen_cranelift/test.sh b/compiler/rustc_codegen_cranelift/test.sh new file mode 100755 index 00000000000..6357eebf026 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec ./y.sh test "$@" diff --git a/compiler/rustc_codegen_cranelift/y.cmd b/compiler/rustc_codegen_cranelift/y.cmd new file mode 100644 index 00000000000..42106849163 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/y.cmd @@ -0,0 +1,7 @@ +@echo off +echo [BUILD] build system >&2 +cargo run --manifest-path build_system/Cargo.toml -- %* || goto :error +goto :EOF + +:error +exit /b diff --git a/compiler/rustc_codegen_cranelift/y.ps1 b/compiler/rustc_codegen_cranelift/y.ps1 new file mode 100755 index 00000000000..821f0ec6e57 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/y.ps1 @@ -0,0 +1,7 @@ +$ErrorActionPreference = "Stop" + +$host.ui.WriteErrorLine("[BUILD] build system") +& cargo run --manifest-path build_system/Cargo.toml -- $args +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} diff --git a/compiler/rustc_codegen_cranelift/y.sh b/compiler/rustc_codegen_cranelift/y.sh new file mode 100755 index 00000000000..b9152d2cc6d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/y.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e +echo "[BUILD] build system" 1>&2 +exec cargo run --manifest-path build_system/Cargo.toml -- "$@" |