Merge branch 'yusdacra/crate2nix' into 'master'

nix: Switch to crate2nix for building, and niv for dependency management

See merge request veloren/veloren!1090

.gitignore

@@ -42,4 +42,7 @@ todo.txt
 *.bat
 
 # Mac
-.DS_Store
+.DS_Store
+
+# Nix
+nix/result

common/build.rs

@@ -7,6 +7,13 @@ use std::{
 };
 
 fn main() {
+    // If the env variable exists then we are building on nix, use it as hash and
+    // return. `git-lfs` check is handled by nix itself.
+    if let Some(hash) = option_env!("NIX_GIT_HASH") {
+        create_hash_file(hash);
+        return;
+    }
+
     // Get the current githash
     // Note: It will compare commits. As long as the commits do not diverge from the
     // server no version change will be detected.
@@ -23,16 +30,7 @@ fn main() {
     {
         Ok(output) => match String::from_utf8(output.stdout) {
             Ok(hash) => {
-                let mut target = File::create(
+                create_hash_file(&hash);
-                    Path::new(
-                        &env::var("OUT_DIR").expect("failed to query OUT_DIR environment variable"),
-                    )
-                    .join("githash"),
-                )
-                .expect("failed to create git hash file!");
-                target
-                    .write_all(hash.trim().as_bytes())
-                    .expect("failed to write to file!");
             },
             Err(e) => panic!("failed to convert git output to UTF-8: {}", e),
         },
@@ -70,3 +68,14 @@ fn main() {
         }
     }
 }
+
+fn create_hash_file(hash: &str) {
+    let mut target = File::create(
+        Path::new(&env::var("OUT_DIR").expect("failed to query OUT_DIR environment variable"))
+            .join("githash"),
+    )
+    .expect("failed to create git hash file!");
+    target
+        .write_all(hash.trim().as_bytes())
+        .expect("failed to write to file!");
+}

default.nix

@@ -1,73 +0,0 @@
-let
-  fallbackPkgs = import <nixpkgs> {};
-  fallbackMozSrc = builtins.fetchTarball "https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz";
-in
-
-{
-  alsaLib ? fallbackPkgs.alsaLib,
-  atk ? fallbackPkgs.atk,
-  cairo ? fallbackPkgs.cairo,
-  git ? null,
-  git-lfs ? null,
-  glib ? fallbackPkgs.glib,
-  gnuplot ? null,
-  gtk3 ? fallbackPkgs.gtk3,
-  libudev ? fallbackPkgs.libudev,
-  makeRustPlatform ? fallbackPkgs.makeRustPlatform,
-  mozSrc ? fallbackMozSrc,
-  nix-gitignore ? fallbackPkgs.nix-gitignore,
-  openssl ? fallbackPkgs.openssl,
-  pango ? fallbackPkgs.pango,
-  pkg-config ? fallbackPkgs.pkg-config,
-  pkgs ? fallbackPkgs,
-  python3 ? fallbackPkgs.python3,
-  rustup ? null,
-  stdenv ? fallbackPkgs.stdenv,
-  veloren-src ? null,
-}:
-
-let
-  # `mozPkgs` is the package set of `mozRepo`; this differs from their README
-  # where they use it as an overlay rather than a separate package set
-  mozPkgs = import "${mozSrc}/package-set.nix" { inherit pkgs; };
-  channel = mozPkgs.rustChannelOf { rustToolchain = ./rust-toolchain; };
-  rustPlatform = makeRustPlatform {
-    rustc = channel.rust;
-    cargo = channel.cargo;
-  };
-in
-
-rustPlatform.buildRustPackage rec {
-  pname = "veloren";
-  version = "unstable";
-  # For information on how to automatically fetch the source from GitLab, please
-  # ask @haslersn
-  src = if veloren-src == null then (nix-gitignore.gitignoreSource [] ./.) else veloren-src;
-  nativeBuildInputs = [
-    pkg-config
-    python3
-    # Convenience for nix-shell
-    git
-    git-lfs
-    gnuplot
-    rustup # Required for integration in some editors
-  ];
-  buildInputs = [
-    alsaLib
-    atk
-    cairo
-    glib
-    gtk3
-    pango
-    libudev
-    openssl
-  ];
-  #preConfigure = "export HOME=`mktemp -d`";
-  postInstall = "cp -R $src/assets $out/bin/assets";
-  # If veloren-vendor build fails with hash mismatch, change this hash with `got:` hash
-  cargoSha256 = "13aa2jypqhg4y7bpkxqdchd0sw85hq6galafswbg1d4bjwphnq70";
-
-  meta = {
-    platforms = stdenv.lib.platforms.linux;
-  };
-}

nix/README.md

@@ -0,0 +1,34 @@
+### How to use
+
+To build Voxygen, run:
+`nix build`
+
+To build another binary, run:
+`nix build --arg crateName "<binary name here>"`
+
+To enter the development shell (which includes all tools mentioned in this readme), run:
+`nix-shell shell.nix`
+
+### Managing Cargo.nix
+
+Enter the development shell.
+
+To update `Cargo.nix` (and `crate-hashes.json`) using latest `Cargo.lock`, run:
+`crate2nix generate -f ../Cargo.toml`
+
+### Managing dependencies
+
+We use [niv](https://github.com/nmattia/niv) to manage dependencies.
+
+Enter the development shell in repository root:
+`cd .. && nix-shell nix/shell.nix`
+
+To update the dependencies, run:
+`niv update`
+
+### Formatting
+
+Use [nixfmt](https://github.com/serokell/nixfmt) to format files.
+
+To format every file:
+`nixfmt *.nix`

nix/crate-hashes.json

@@ -0,0 +1,13 @@
+{
+  "auth-common 0.1.0 (git+https://gitlab.com/veloren/auth.git?rev=223a4097f7ebc8d451936dccb5e6517194bbf086#223a4097f7ebc8d451936dccb5e6517194bbf086)": "1b0ipil3b7kxss5wi7ij51wncbww9cdr5nydrh1dwd787agykrai",
+  "authc 1.0.0 (git+https://gitlab.com/veloren/auth.git?rev=223a4097f7ebc8d451936dccb5e6517194bbf086#223a4097f7ebc8d451936dccb5e6517194bbf086)": "1b0ipil3b7kxss5wi7ij51wncbww9cdr5nydrh1dwd787agykrai",
+  "conrod_core 0.63.0 (git+https://gitlab.com/veloren/conrod.git?branch=pre-winit-20#46b374edc9537300e5278905ebd14dff45cfd927)": "0lrh3v8dwr9x01qjry7p4wkcvd9r2cvn2865fa5nrbrc77d1qjkn",
+  "conrod_derive 0.63.0 (git+https://gitlab.com/veloren/conrod.git?branch=pre-winit-20#46b374edc9537300e5278905ebd14dff45cfd927)": "0lrh3v8dwr9x01qjry7p4wkcvd9r2cvn2865fa5nrbrc77d1qjkn",
+  "conrod_winit 0.63.0 (git+https://gitlab.com/veloren/conrod.git?branch=pre-winit-20#46b374edc9537300e5278905ebd14dff45cfd927)": "0lrh3v8dwr9x01qjry7p4wkcvd9r2cvn2865fa5nrbrc77d1qjkn",
+  "euc 0.5.1 (git+https://github.com/zesterer/euc.git#c9a7c17a03d45fce00caeeca09afa1e1558cd183)": "0qvk0bx1arkgmdc59sip39zszdw2fwv6jcy5jinv18n5m1nrclw4",
+  "guillotiere 0.4.2 (git+https://github.com/Imberflur/guillotiere#42c298f5bcf0f95f1a004360d05e25ca3711e9ed)": "1knqbn777f3cgzbsaqawzclgrqf3nav6x3gjzc2hsls3acccx1ya",
+  "msgbox 0.4.0 (git+https://github.com/bekker/msgbox-rs.git?rev=68fe39a#68fe39a60019b38a1569ae4e9ed796a0f0542673)": "18h6s50n7mz2ggfishhi91p2298shqhzx5xagpg9q4zb4y90c2wr",
+  "portpicker 0.1.0 (git+https://github.com/xMAC94x/portpicker-rs#9d6df36c53c94684080a64a7212dd6bfc3617ee4)": "00vl2k3xfihxq86kf5rsknjl8dxmrxqhwajwn0hj4gzgnbssr0rx",
+  "specs-idvs 0.1.0 (git+https://gitlab.com/veloren/specs-idvs.git#111548cf44e9bb4c43feb12aa3fc253953ac6e4a)": "1qgbzr4g4iys10hi56l6z7k3gk9qwj9xamfdigqnkhxvcynfhqqv",
+  "treeculler 0.1.0 (git+https://gitlab.com/yusdacra/treeculler.git#efcf5283cf386117a7e654abdaa45ef664a08e42)": "19niwgha0jnvrp22pq0070dfimb2wkda53a3parhga3vhap2g01b"
+}

nix/default.nix

@@ -0,0 +1,50 @@
+{ crateName ? "veloren-voxygen",
+# `crate2nix` doesn't support profiles in `Cargo.toml`, so default to release. Otherwise bad performance (non-release is built with opt level 0)
+release ? true, sources ? import ./sources.nix { }, nixpkgsSrc ? <nixpkgs> }:
+
+let
+  # Check if git-lfs is working.
+  isGitLfsSetup =
+    if builtins.pathExists ../assets/voxygen/background/bg_main.png then
+      true
+    else
+      abort ''
+        Git Large File Storage (git-lfs) has not been set up correctly.
+        Most common reasons:
+        	- git-lfs was not installed before cloning this repository
+        	- this repository was not cloned from the primary gitlab mirror.
+        	- The github mirror does not support lfs.
+        See the book at https://book.veloren.net/ for details.
+      '';
+
+  pkgs = import ./nixpkgs.nix { inherit sources nixpkgsSrc; };
+
+  # Only copy the `.git` directory to nix store, anything else is a waste.
+  gitSrc = builtins.path {
+    path = ../.git;
+    name = "git";
+  };
+  gitHash = builtins.readFile (with pkgs;
+    runCommand "getGitHash" { nativeBuildInputs = [ git ]; } ''
+      cd ${gitSrc}
+      git log -n 1 --pretty=format:%h/%cd --date=format:%Y-%m-%d-%H:%M --abbrev=8 > $out
+    '');
+
+  veloren = with pkgs;
+    callPackage ./Cargo.nix {
+      defaultCrateOverrides = defaultCrateOverrides // {
+        libudev-sys = attrs: { buildInputs = [ pkg-config libudev ]; };
+        alsa-sys = attrs: { buildInputs = [ pkg-config alsaLib ]; };
+        veloren-common = attrs: {
+          NIX_GIT_HASH = gitHash;
+          # We need to include the result here otherwise nix won't evaluate the check.
+          GIT_LFS_SETUP = isGitLfsSetup;
+        };
+        veloren-network = attrs: { buildInputs = [ pkg-config openssl ]; };
+        veloren-voxygen = attrs: {
+          buildInputs = [ atk cairo glib gtk3 pango ];
+        };
+      };
+      inherit release pkgs;
+    };
+in veloren.workspaceMembers."${crateName}".build

nix/nixpkgs.nix

@@ -0,0 +1,15 @@
+{ sources ? import ./sources.nix { }, nixpkgsSrc ? <nixpkgs> }:
+
+let
+  mozPkgs = import "${sources.nixpkgsMoz}/package-set.nix" {
+    pkgs = import nixpkgsSrc { };
+  };
+  rustChannel = mozPkgs.rustChannelOf { rustToolchain = ../rust-toolchain; };
+in import nixpkgsSrc {
+  overlays = [
+    (self: super: {
+      rustc = rustChannel.rust;
+      inherit (rustChannel) cargo rust rust-std rust-src;
+    })
+  ];
+}

nix/shell.nix

@@ -0,0 +1,26 @@
+{ sources ? import ./sources.nix { }, pkgs ? import <nixpkgs> { } }:
+
+let crate2nix = import sources.crate2nix { };
+in pkgs.mkShell {
+  name = "veloren-shell";
+  nativeBuildInputs = with pkgs; [
+    pkg-config
+    python3
+    git
+    git-lfs
+    niv
+    nixfmt
+    crate2nix
+    rustup
+  ];
+  buildInputs = with pkgs; [
+    alsaLib
+    atk
+    cairo
+    glib
+    gtk3
+    libudev
+    openssl
+    pango
+  ];
+}

nix/sources.json

@@ -0,0 +1,26 @@
+{
+    "crate2nix": {
+        "branch": "master",
+        "description": "nix build file generator for rust crates",
+        "homepage": "",
+        "owner": "kolloch",
+        "repo": "crate2nix",
+        "rev": "87a544222723693c81f3b4141d639ba9b5769e2e",
+        "sha256": "0rml8xy7b6qlx631wpa29yb8qjsilv55w6g5rqbqjyw22aqz4ppb",
+        "type": "tarball",
+        "url": "https://github.com/kolloch/crate2nix/archive/87a544222723693c81f3b4141d639ba9b5769e2e.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgsMoz": {
+        "branch": "master",
+        "description": "mozilla related nixpkgs (extends nixos/nixpkgs repo)",
+        "homepage": null,
+        "owner": "mozilla",
+        "repo": "nixpkgs-mozilla",
+        "rev": "e912ed483e980dfb4666ae0ed17845c4220e5e7c",
+        "sha256": "08fvzb8w80bkkabc1iyhzd15f4sm7ra10jn32kfch5klgl0gj3j3",
+        "type": "tarball",
+        "url": "https://github.com/mozilla/nixpkgs-mozilla/archive/e912ed483e980dfb4666ae0ed17845c4220e5e7c.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}

nix/sources.nix

@@ -0,0 +1,154 @@
+# This file has been generated by Niv.
+
+let
+
+  #
+  # The fetchers. fetch_<type> fetches specs of type <type>.
+  #
+
+  fetch_file = pkgs: spec:
+    if spec.builtin or true then
+      builtins_fetchurl { inherit (spec) url sha256; }
+    else
+      pkgs.fetchurl { inherit (spec) url sha256; };
+
+  fetch_tarball = pkgs: name: spec:
+    let
+      ok = str: !builtins.isNull (builtins.match "[a-zA-Z0-9+-._?=]" str);
+      # sanitize the name, though nix will still fail if name starts with period
+      name' = stringAsChars (x: if !ok x then "-" else x) "${name}-src";
+    in if spec.builtin or true then
+      builtins_fetchTarball {
+        name = name';
+        inherit (spec) url sha256;
+      }
+    else
+      pkgs.fetchzip {
+        name = name';
+        inherit (spec) url sha256;
+      };
+
+  fetch_git = spec:
+    builtins.fetchGit {
+      url = spec.repo;
+      inherit (spec) rev ref;
+    };
+
+  fetch_local = spec: spec.path;
+
+  fetch_builtin-tarball = name:
+    throw ''
+      [${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
+              $ niv modify ${name} -a type=tarball -a builtin=true'';
+
+  fetch_builtin-url = name:
+    throw ''
+      [${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
+              $ niv modify ${name} -a type=file -a builtin=true'';
+
+  #
+  # Various helpers
+  #
+
+  # The set of packages used when specs are fetched using non-builtins.
+  mkPkgs = sources:
+    let
+      sourcesNixpkgs =
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; })
+        { };
+      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
+      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
+    in if builtins.hasAttr "nixpkgs" sources then
+      sourcesNixpkgs
+    else if hasNixpkgsPath && !hasThisAsNixpkgsPath then
+      import <nixpkgs> { }
+    else
+      abort ''
+        Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+        add a package called "nixpkgs" to your sources.json.
+      '';
+
+  # The actual fetching function.
+  fetch = pkgs: name: spec:
+
+    if !builtins.hasAttr "type" spec then
+      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
+    else if spec.type == "file" then
+      fetch_file pkgs spec
+    else if spec.type == "tarball" then
+      fetch_tarball pkgs name spec
+    else if spec.type == "git" then
+      fetch_git spec
+    else if spec.type == "local" then
+      fetch_local spec
+    else if spec.type == "builtin-tarball" then
+      fetch_builtin-tarball name
+    else if spec.type == "builtin-url" then
+      fetch_builtin-url name
+    else
+      abort
+      "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
+
+  # Ports of functions for older nix versions
+
+  # a Nix version of mapAttrs if the built-in doesn't exist
+  mapAttrs = builtins.mapAttrs or (f: set:
+    with builtins;
+    listToAttrs (map (attr: {
+      name = attr;
+      value = f attr set.${attr};
+    }) (attrNames set)));
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
+  range = first: last:
+    if first > last then
+      [ ]
+    else
+      builtins.genList (n: first + n) (last - first + 1);
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
+  stringToCharacters = s:
+    map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
+  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
+  concatStrings = builtins.concatStringsSep "";
+
+  # fetchTarball version that is compatible between all the versions of Nix
+  builtins_fetchTarball = { url, name, sha256 }@attrs:
+    let inherit (builtins) lessThan nixVersion fetchTarball;
+    in if lessThan nixVersion "1.12" then
+      fetchTarball { inherit name url; }
+    else
+      fetchTarball attrs;
+
+  # fetchurl version that is compatible between all the versions of Nix
+  builtins_fetchurl = { url, sha256 }@attrs:
+    let inherit (builtins) lessThan nixVersion fetchurl;
+    in if lessThan nixVersion "1.12" then
+      fetchurl { inherit url; }
+    else
+      fetchurl attrs;
+
+  # Create the final "sources" from the config
+  mkSources = config:
+    mapAttrs (name: spec:
+      if builtins.hasAttr "outPath" spec then
+        abort
+        "The values in sources.json should not have an 'outPath' attribute"
+      else
+        spec // { outPath = fetch config.pkgs name spec; }) config.sources;
+
+  # The "config" used by the fetchers
+  mkConfig = { sourcesFile ? ./sources.json
+    , sources ? builtins.fromJSON (builtins.readFile sourcesFile)
+    , pkgs ? mkPkgs sources }: rec {
+      # The sources, i.e. the attribute set of spec name to spec
+      inherit sources;
+
+      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
+      inherit pkgs;
+    };
+in mkSources (mkConfig { }) // {
+  __functor = _: settings: mkSources (mkConfig settings);
+}

shell.nix

@@ -1,34 +0,0 @@
-with import <nixpkgs> {};
-
-import ./default.nix {
-  git = git;
-  git-lfs = git-lfs;
-  gnuplot = gnuplot;
-  rustup = rustup;
-  # The source is copied to the nix store. We don't want to do this (including assets) for every
-  # time the `nix-shell` is entered. Therefore we create a source which contains only the files
-  # necessary to evaluate `buildRustPackage` successfully:
-  veloren-src = runCommand "veloren-shell" {} (lib.concatMapStrings
-    (p: "mkdir -p $(dirname $out/${p}); cp ${./. + "/${p}"} $out/${p}\n")
-    [
-      "Cargo.lock"
-      "Cargo.toml"
-      "chat-cli/Cargo.toml"
-      "chat-cli/src/main.rs"
-      "client/Cargo.toml"
-      "client/src/lib.rs"
-      "common/Cargo.toml"
-      "common/src/lib.rs"
-      "server-cli/Cargo.toml"
-      "server-cli/src/main.rs"
-      "server/Cargo.toml"
-      "server/src/lib.rs"
-      "voxygen/Cargo.toml"
-      "voxygen/src/main.rs"
-      "world/Cargo.toml"
-      "world/src/lib.rs"
-      "network/Cargo.toml"
-      "network/src/lib.rs"
-    ]
-  );
-}