Auto merge of #62452 - Centril:rollup-5jww3h7, r=Centril

Rollup of 5 pull requests

Successful merges:

 - #60081 (Refactor unicode.py script)
 - #61862 (Make the Weak::{into,as}_raw methods)
 - #62243 (Improve documentation for built-in macros)
 - #62422 (Remove some uses of mem::uninitialized)
 - #62436 (normalize use of backticks/lowercase in compiler messages for librustc_mir)

Failed merges:

r? @ghost

3 files changed, 1369 insertions, 443 deletions

diff --git a/src/libcore/lib.rs b/src/libcore/lib.rs
index 6ab64a11237..782627431cb 100644
--- a/src/libcore/lib.rs
+++ b/src/libcore/lib.rs
@@ -75,6 +75,7 @@
 #![feature(const_fn)]
 #![feature(const_fn_union)]
 #![feature(custom_inner_attributes)]
+#![feature(decl_macro)]
 #![feature(doc_cfg)]
 #![feature(doc_spotlight)]
 #![feature(extern_types)]
diff --git a/src/libcore/macros.rs b/src/libcore/macros.rs
index 589061b2826..33ffd84e521 100644
--- a/src/libcore/macros.rs
+++ b/src/libcore/macros.rs
@@ -642,183 +642,720 @@ macro_rules! uninitialized_array {
 /// These macros do not have any corresponding definition with a `macro_rules!`
 /// macro, but are documented here. Their implementations can be found hardcoded
 /// into libsyntax itself.
-///
-/// For more information, see documentation for `std`'s macros.
 #[cfg(rustdoc)]
 mod builtin {
 
     /// Causes compilation to fail with the given error message when encountered.
     ///
-    /// For more information, see the documentation for [`std::compile_error!`].
+    /// This macro should be used when a crate uses a conditional compilation strategy to provide
+    /// better error messages for erroneous conditions. It's the compiler-level form of [`panic!`],
+    /// which emits an error at *runtime*, rather than during compilation.
+    ///
+    /// # Examples
+    ///
+    /// Two such examples are macros and `#[cfg]` environments.
+    ///
+    /// Emit better compiler error if a macro is passed invalid values. Without the final branch,
+    /// the compiler would still emit an error, but the error's message would not mention the two
+    /// valid values.
+    ///
+    /// ```compile_fail
+    /// macro_rules! give_me_foo_or_bar {
+    ///     (foo) => {};
+    ///     (bar) => {};
+    ///     ($x:ident) => {
+    ///         compile_error!("This macro only accepts `foo` or `bar`");
+    ///     }
+    /// }
+    ///
+    /// give_me_foo_or_bar!(neither);
+    /// // ^ will fail at compile time with message "This macro only accepts `foo` or `bar`"
+    /// ```
+    ///
+    /// Emit compiler error if one of a number of features isn't available.
+    ///
+    /// ```compile_fail
+    /// #[cfg(not(any(feature = "foo", feature = "bar")))]
+    /// compile_error!("Either feature \"foo\" or \"bar\" must be enabled for this crate.");
+    /// ```
     ///
-    /// [`std::compile_error!`]: ../std/macro.compile_error.html
+    /// [`panic!`]: ../std/macro.panic.html
     #[stable(feature = "compile_error_macro", since = "1.20.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! compile_error {
-        ($msg:expr) => ({ /* compiler built-in */ });
-        ($msg:expr,) => ({ /* compiler built-in */ });
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro compile_error {
+        ($msg:expr) => ({ /* compiler built-in */ }),
+        ($msg:expr,) => ({ /* compiler built-in */ })
     }
 
     /// Constructs parameters for the other string-formatting macros.
     ///
-    /// For more information, see the documentation for [`std::format_args!`].
+    /// This macro functions by taking a formatting string literal containing
+    /// `{}` for each additional argument passed. `format_args!` prepares the
+    /// additional parameters to ensure the output can be interpreted as a string
+    /// and canonicalizes the arguments into a single type. Any value that implements
+    /// the [`Display`] trait can be passed to `format_args!`, as can any
+    /// [`Debug`] implementation be passed to a `{:?}` within the formatting string.
+    ///
+    /// This macro produces a value of type [`fmt::Arguments`]. This value can be
+    /// passed to the macros within [`std::fmt`] for performing useful redirection.
+    /// All other formatting macros ([`format!`], [`write!`], [`println!`], etc) are
+    /// proxied through this one. `format_args!`, unlike its derived macros, avoids
+    /// heap allocations.
+    ///
+    /// You can use the [`fmt::Arguments`] value that `format_args!` returns
+    /// in `Debug` and `Display` contexts as seen below. The example also shows
+    /// that `Debug` and `Display` format to the same thing: the interpolated
+    /// format string in `format_args!`.
+    ///
+    /// ```rust
+    /// let debug = format!("{:?}", format_args!("{} foo {:?}", 1, 2));
+    /// let display = format!("{}", format_args!("{} foo {:?}", 1, 2));
+    /// assert_eq!("1 foo 2", display);
+    /// assert_eq!(display, debug);
+    /// ```
+    ///
+    /// For more information, see the documentation in [`std::fmt`].
+    ///
+    /// [`Display`]: ../std/fmt/trait.Display.html
+    /// [`Debug`]: ../std/fmt/trait.Debug.html
+    /// [`fmt::Arguments`]: ../std/fmt/struct.Arguments.html
+    /// [`std::fmt`]: ../std/fmt/index.html
+    /// [`format!`]: ../std/macro.format.html
+    /// [`write!`]: ../std/macro.write.html
+    /// [`println!`]: ../std/macro.println.html
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::fmt;
     ///
-    /// [`std::format_args!`]: ../std/macro.format_args.html
+    /// let s = fmt::format(format_args!("hello {}", "world"));
+    /// assert_eq!(s, format!("hello {}", "world"));
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! format_args {
-        ($fmt:expr) => ({ /* compiler built-in */ });
-        ($fmt:expr, $($args:tt)*) => ({ /* compiler built-in */ });
+    #[allow_internal_unstable(fmt_internals)]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro format_args {
+        ($fmt:expr) => ({ /* compiler built-in */ }),
+        ($fmt:expr, $($args:tt)*) => ({ /* compiler built-in */ })
+    }
+
+    /// Same as `format_args`, but adds a newline in the end.
+    #[unstable(feature = "format_args_nl", issue = "0",
+               reason = "`format_args_nl` is only for internal \
+                         language use and is subject to change")]
+    #[allow_internal_unstable(fmt_internals)]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro format_args_nl {
+        ($fmt:expr) => ({ /* compiler built-in */ }),
+        ($fmt:expr, $($args:tt)*) => ({ /* compiler built-in */ })
     }
 
     /// Inspects an environment variable at compile time.
     ///
-    /// For more information, see the documentation for [`std::env!`].
+    /// This macro will expand to the value of the named environment variable at
+    /// compile time, yielding an expression of type `&'static str`.
+    ///
+    /// If the environment variable is not defined, then a compilation error
+    /// will be emitted. To not emit a compile error, use the [`option_env!`]
+    /// macro instead.
+    ///
+    /// [`option_env!`]: ../std/macro.option_env.html
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let path: &'static str = env!("PATH");
+    /// println!("the $PATH variable at the time of compiling was: {}", path);
+    /// ```
+    ///
+    /// You can customize the error message by passing a string as the second
+    /// parameter:
+    ///
+    /// ```compile_fail
+    /// let doc: &'static str = env!("documentation", "what's that?!");
+    /// ```
     ///
-    /// [`std::env!`]: ../std/macro.env.html
+    /// If the `documentation` environment variable is not defined, you'll get
+    /// the following error:
+    ///
+    /// ```text
+    /// error: what's that?!
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! env {
-        ($name:expr) => ({ /* compiler built-in */ });
-        ($name:expr,) => ({ /* compiler built-in */ });
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro env {
+        ($name:expr) => ({ /* compiler built-in */ }),
+        ($name:expr,) => ({ /* compiler built-in */ })
     }
 
     /// Optionally inspects an environment variable at compile time.
     ///
-    /// For more information, see the documentation for [`std::option_env!`].
+    /// If the named environment variable is present at compile time, this will
+    /// expand into an expression of type `Option<&'static str>` whose value is
+    /// `Some` of the value of the environment variable. If the environment
+    /// variable is not present, then this will expand to `None`. See
+    /// [`Option<T>`][option] for more information on this type.
+    ///
+    /// A compile time error is never emitted when using this macro regardless
+    /// of whether the environment variable is present or not.
+    ///
+    /// [option]: ../std/option/enum.Option.html
+    ///
+    /// # Examples
     ///
-    /// [`std::option_env!`]: ../std/macro.option_env.html
+    /// ```
+    /// let key: Option<&'static str> = option_env!("SECRET_KEY");
+    /// println!("the secret key might be: {:?}", key);
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! option_env {
-        ($name:expr) => ({ /* compiler built-in */ });
-        ($name:expr,) => ({ /* compiler built-in */ });
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro option_env {
+        ($name:expr) => ({ /* compiler built-in */ }),
+        ($name:expr,) => ({ /* compiler built-in */ })
     }
 
     /// Concatenates identifiers into one identifier.
     ///
-    /// For more information, see the documentation for [`std::concat_idents!`].
+    /// This macro takes any number of comma-separated identifiers, and
+    /// concatenates them all into one, yielding an expression which is a new
+    /// identifier. Note that hygiene makes it such that this macro cannot
+    /// capture local variables. Also, as a general rule, macros are only
+    /// allowed in item, statement or expression position. That means while
+    /// you may use this macro for referring to existing variables, functions or
+    /// modules etc, you cannot define a new one with it.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(concat_idents)]
+    ///
+    /// # fn main() {
+    /// fn foobar() -> u32 { 23 }
     ///
-    /// [`std::concat_idents!`]: ../std/macro.concat_idents.html
-    #[unstable(feature = "concat_idents_macro", issue = "29599")]
-    #[rustc_doc_only_macro]
-    macro_rules! concat_idents {
-        ($($e:ident),+) => ({ /* compiler built-in */ });
-        ($($e:ident,)+) => ({ /* compiler built-in */ });
+    /// let f = concat_idents!(foo, bar);
+    /// println!("{}", f());
+    ///
+    /// // fn concat_idents!(new, fun, name) { } // not usable in this way!
+    /// # }
+    /// ```
+    #[unstable(feature = "concat_idents", issue = "29599",
+               reason = "`concat_idents` is not stable enough for use and is subject to change")]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro concat_idents {
+        ($($e:ident),+) => ({ /* compiler built-in */ }),
+        ($($e:ident,)+) => ({ /* compiler built-in */ })
     }
 
     /// Concatenates literals into a static string slice.
     ///
-    /// For more information, see the documentation for [`std::concat!`].
+    /// This macro takes any number of comma-separated literals, yielding an
+    /// expression of type `&'static str` which represents all of the literals
+    /// concatenated left-to-right.
+    ///
+    /// Integer and floating point literals are stringified in order to be
+    /// concatenated.
     ///
-    /// [`std::concat!`]: ../std/macro.concat.html
+    /// # Examples
+    ///
+    /// ```
+    /// let s = concat!("test", 10, 'b', true);
+    /// assert_eq!(s, "test10btrue");
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! concat {
-        ($($e:expr),*) => ({ /* compiler built-in */ });
-        ($($e:expr,)*) => ({ /* compiler built-in */ });
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro concat {
+        ($($e:expr),*) => ({ /* compiler built-in */ }),
+        ($($e:expr,)*) => ({ /* compiler built-in */ })
     }
 
     /// Expands to the line number on which it was invoked.
     ///
-    /// For more information, see the documentation for [`std::line!`].
+    /// With [`column!`] and [`file!`], these macros provide debugging information for
+    /// developers about the location within the source.
+    ///
+    /// The expanded expression has type `u32` and is 1-based, so the first line
+    /// in each file evaluates to 1, the second to 2, etc. This is consistent
+    /// with error messages by common compilers or popular editors.
+    /// The returned line is *not necessarily* the line of the `line!` invocation itself,
+    /// but rather the first macro invocation leading up to the invocation
+    /// of the `line!` macro.
     ///
-    /// [`std::line!`]: ../std/macro.line.html
+    /// [`column!`]: macro.column.html
+    /// [`file!`]: macro.file.html
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let current_line = line!();
+    /// println!("defined on line: {}", current_line);
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! line { () => ({ /* compiler built-in */ }) }
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro line() { /* compiler built-in */ }
 
-    /// Expands to the column number on which it was invoked.
+    /// Expands to the column number at which it was invoked.
+    ///
+    /// With [`line!`] and [`file!`], these macros provide debugging information for
+    /// developers about the location within the source.
+    ///
+    /// The expanded expression has type `u32` and is 1-based, so the first column
+    /// in each line evaluates to 1, the second to 2, etc. This is consistent
+    /// with error messages by common compilers or popular editors.
+    /// The returned column is *not necessarily* the line of the `column!` invocation itself,
+    /// but rather the first macro invocation leading up to the invocation
+    /// of the `column!` macro.
     ///
-    /// For more information, see the documentation for [`std::column!`].
+    /// [`line!`]: macro.line.html
+    /// [`file!`]: macro.file.html
     ///
-    /// [`std::column!`]: ../std/macro.column.html
+    /// # Examples
+    ///
+    /// ```
+    /// let current_col = column!();
+    /// println!("defined on column: {}", current_col);
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! column { () => ({ /* compiler built-in */ }) }
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro column() { /* compiler built-in */ }
+
+    /// Same as `column`, but less likely to be shadowed.
+    #[unstable(feature = "__rust_unstable_column", issue = "0",
+               reason = "internal implementation detail of the `column` macro")]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro __rust_unstable_column() { /* compiler built-in */ }
 
-    /// Expands to the file name from which it was invoked.
+    /// Expands to the file name in which it was invoked.
+    ///
+    /// With [`line!`] and [`column!`], these macros provide debugging information for
+    /// developers about the location within the source.
+    ///
     ///
-    /// For more information, see the documentation for [`std::file!`].
+    /// The expanded expression has type `&'static str`, and the returned file
+    /// is not the invocation of the `file!` macro itself, but rather the
+    /// first macro invocation leading up to the invocation of the `file!`
+    /// macro.
     ///
-    /// [`std::file!`]: ../std/macro.file.html
+    /// [`line!`]: macro.line.html
+    /// [`column!`]: macro.column.html
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let this_file = file!();
+    /// println!("defined in file: {}", this_file);
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! file { () => ({ /* compiler built-in */ }) }
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro file() { /* compiler built-in */ }
 
     /// Stringifies its arguments.
     ///
-    /// For more information, see the documentation for [`std::stringify!`].
+    /// This macro will yield an expression of type `&'static str` which is the
+    /// stringification of all the tokens passed to the macro. No restrictions
+    /// are placed on the syntax of the macro invocation itself.
+    ///
+    /// Note that the expanded results of the input tokens may change in the
+    /// future. You should be careful if you rely on the output.
     ///
-    /// [`std::stringify!`]: ../std/macro.stringify.html
+    /// # Examples
+    ///
+    /// ```
+    /// let one_plus_one = stringify!(1 + 1);
+    /// assert_eq!(one_plus_one, "1 + 1");
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! stringify { ($($t:tt)*) => ({ /* compiler built-in */ }) }
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro stringify($($t:tt)*) { /* compiler built-in */ }
 
     /// Includes a utf8-encoded file as a string.
     ///
-    /// For more information, see the documentation for [`std::include_str!`].
+    /// The file is located relative to the current file. (similarly to how
+    /// modules are found)
+    ///
+    /// This macro will yield an expression of type `&'static str` which is the
+    /// contents of the file.
+    ///
+    /// # Examples
+    ///
+    /// Assume there are two files in the same directory with the following
+    /// contents:
+    ///
+    /// File 'spanish.in':
+    ///
+    /// ```text
+    /// adiós
+    /// ```
+    ///
+    /// File 'main.rs':
+    ///
+    /// ```ignore (cannot-doctest-external-file-dependency)
+    /// fn main() {
+    ///     let my_str = include_str!("spanish.in");
+    ///     assert_eq!(my_str, "adiós\n");
+    ///     print!("{}", my_str);
+    /// }
+    /// ```
     ///
-    /// [`std::include_str!`]: ../std/macro.include_str.html
+    /// Compiling 'main.rs' and running the resulting binary will print "adiós".
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! include_str {
-        ($file:expr) => ({ /* compiler built-in */ });
-        ($file:expr,) => ({ /* compiler built-in */ });
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro include_str {
+        ($file:expr) => ({ /* compiler built-in */ }),
+        ($file:expr,) => ({ /* compiler built-in */ })
     }
 
     /// Includes a file as a reference to a byte array.
     ///
-    /// For more information, see the documentation for [`std::include_bytes!`].
+    /// The file is located relative to the current file. (similarly to how
+    /// modules are found)
+    ///
+    /// This macro will yield an expression of type `&'static [u8; N]` which is
+    /// the contents of the file.
+    ///
+    /// # Examples
+    ///
+    /// Assume there are two files in the same directory with the following
+    /// contents:
+    ///
+    /// File 'spanish.in':
+    ///
+    /// ```text
+    /// adiós
+    /// ```
+    ///
+    /// File 'main.rs':
+    ///
+    /// ```ignore (cannot-doctest-external-file-dependency)
+    /// fn main() {
+    ///     let bytes = include_bytes!("spanish.in");
+    ///     assert_eq!(bytes, b"adi\xc3\xb3s\n");
+    ///     print!("{}", String::from_utf8_lossy(bytes));
+    /// }
+    /// ```
     ///
-    /// [`std::include_bytes!`]: ../std/macro.include_bytes.html
+    /// Compiling 'main.rs' and running the resulting binary will print "adiós".
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! include_bytes {
-        ($file:expr) => ({ /* compiler built-in */ });
-        ($file:expr,) => ({ /* compiler built-in */ });
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro include_bytes {
+        ($file:expr) => ({ /* compiler built-in */ }),
+        ($file:expr,) => ({ /* compiler built-in */ })
     }
 
     /// Expands to a string that represents the current module path.
     ///
-    /// For more information, see the documentation for [`std::module_path!`].
+    /// The current module path can be thought of as the hierarchy of modules
+    /// leading back up to the crate root. The first component of the path
+    /// returned is the name of the crate currently being compiled.
     ///
-    /// [`std::module_path!`]: ../std/macro.module_path.html
+    /// # Examples
+    ///
+    /// ```
+    /// mod test {
+    ///     pub fn foo() {
+    ///         assert!(module_path!().ends_with("test"));
+    ///     }
+    /// }
+    ///
+    /// test::foo();
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! module_path { () => ({ /* compiler built-in */ }) }
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro module_path() { /* compiler built-in */ }
 
-    /// Evaluates boolean combinations of configuration flags, at compile-time.
+    /// Evaluates boolean combinations of configuration flags at compile-time.
+    ///
+    /// In addition to the `#[cfg]` attribute, this macro is provided to allow
+    /// boolean expression evaluation of configuration flags. This frequently
+    /// leads to less duplicated code.
+    ///
+    /// The syntax given to this macro is the same syntax as the [`cfg`]
+    /// attribute.
     ///
-    /// For more information, see the documentation for [`std::cfg!`].
+    /// [`cfg`]: ../reference/conditional-compilation.html#the-cfg-attribute
     ///
-    /// [`std::cfg!`]: ../std/macro.cfg.html
+    /// # Examples
+    ///
+    /// ```
+    /// let my_directory = if cfg!(windows) {
+    ///     "windows-specific-directory"
+    /// } else {
+    ///     "unix-directory"
+    /// };
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! cfg { ($($cfg:tt)*) => ({ /* compiler built-in */ }) }
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro cfg($($cfg:tt)*) { /* compiler built-in */ }
 
     /// Parses a file as an expression or an item according to the context.
     ///
-    /// For more information, see the documentation for [`std::include!`].
+    /// The file is located relative to the current file (similarly to how
+    /// modules are found).
+    ///
+    /// Using this macro is often a bad idea, because if the file is
+    /// parsed as an expression, it is going to be placed in the
+    /// surrounding code unhygienically. This could result in variables
+    /// or functions being different from what the file expected if
+    /// there are variables or functions that have the same name in
+    /// the current file.
+    ///
+    /// # Examples
+    ///
+    /// Assume there are two files in the same directory with the following
+    /// contents:
     ///
-    /// [`std::include!`]: ../std/macro.include.html
+    /// File 'monkeys.in':
+    ///
+    /// ```ignore (only-for-syntax-highlight)
+    /// ['🙈', '🙊', '🙉']
+    ///     .iter()
+    ///     .cycle()
+    ///     .take(6)
+    ///     .collect::<String>()
+    /// ```
+    ///
+    /// File 'main.rs':
+    ///
+    /// ```ignore (cannot-doctest-external-file-dependency)
+    /// fn main() {
+    ///     let my_string = include!("monkeys.in");
+    ///     assert_eq!("🙈🙊🙉🙈🙊🙉", my_string);
+    ///     println!("{}", my_string);
+    /// }
+    /// ```
+    ///
+    /// Compiling 'main.rs' and running the resulting binary will print
+    /// "🙈🙊🙉🙈🙊🙉".
     #[stable(feature = "rust1", since = "1.0.0")]
-    #[rustc_doc_only_macro]
-    macro_rules! include {
-        ($file:expr) => ({ /* compiler built-in */ });
-        ($file:expr,) => ({ /* compiler built-in */ });
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro include {
+        ($file:expr) => ({ /* compiler built-in */ }),
+        ($file:expr,) => ({ /* compiler built-in */ })
     }
 
     /// Asserts that a boolean expression is `true` at runtime.
     ///
-    /// For more information, see the documentation for [`std::assert!`].
+    /// This will invoke the [`panic!`] macro if the provided expression cannot be
+    /// evaluated to `true` at runtime.
+    ///
+    /// # Uses
+    ///
+    /// Assertions are always checked in both debug and release builds, and cannot
+    /// be disabled. See [`debug_assert!`] for assertions that are not enabled in
+    /// release builds by default.
+    ///
+    /// Unsafe code relies on `assert!` to enforce run-time invariants that, if
+    /// violated could lead to unsafety.
     ///
-    /// [`std::assert!`]: ../std/macro.assert.html
-    #[rustc_doc_only_macro]
+    /// Other use-cases of `assert!` include testing and enforcing run-time
+    /// invariants in safe code (whose violation cannot result in unsafety).
+    ///
+    /// # Custom Messages
+    ///
+    /// This macro has a second form, where a custom panic message can
+    /// be provided with or without arguments for formatting. See [`std::fmt`]
+    /// for syntax for this form.
+    ///
+    /// [`panic!`]: macro.panic.html
+    /// [`debug_assert!`]: macro.debug_assert.html
+    /// [`std::fmt`]: ../std/fmt/index.html
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// // the panic message for these assertions is the stringified value of the
+    /// // expression given.
+    /// assert!(true);
+    ///
+    /// fn some_computation() -> bool { true } // a very simple function
+    ///
+    /// assert!(some_computation());
+    ///
+    /// // assert with a custom message
+    /// let x = true;
+    /// assert!(x, "x wasn't true!");
+    ///
+    /// let a = 3; let b = 27;
+    /// assert!(a + b == 30, "a = {}, b = {}", a, b);
+    /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
-    macro_rules! assert {
-        ($cond:expr) => ({ /* compiler built-in */ });
-        ($cond:expr,) => ({ /* compiler built-in */ });
-        ($cond:expr, $($arg:tt)+) => ({ /* compiler built-in */ });
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro assert {
+        ($cond:expr) => ({ /* compiler built-in */ }),
+        ($cond:expr,) => ({ /* compiler built-in */ }),
+        ($cond:expr, $($arg:tt)+) => ({ /* compiler built-in */ })
     }
+
+    /// Inline assembly.
+    #[unstable(feature = "asm", issue = "29722",
+               reason = "inline assembly is not stable enough for use and is subject to change")]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro asm("assembly template"
+                  : $("output"(operand),)*
+                  : $("input"(operand),)*
+                  : $("clobbers",)*
+                  : $("options",)*) { /* compiler built-in */ }
+
+    /// Module-level inline assembly.
+    #[unstable(feature = "global_asm", issue = "35119",
+               reason = "`global_asm!` is not stable enough for use and is subject to change")]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro global_asm("assembly") { /* compiler built-in */ }
+
+    /// Prints passed tokens into the standard output.
+    #[unstable(feature = "log_syntax", issue = "29598",
+               reason = "`log_syntax!` is not stable enough for use and is subject to change")]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro log_syntax($($arg:tt)*) { /* compiler built-in */ }
+
+    /// Enables or disables tracing functionality used for debugging other macros.
+    #[unstable(feature = "trace_macros", issue = "29598",
+               reason = "`trace_macros` is not stable enough for use and is subject to change")]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro trace_macros {
+        (true) => ({ /* compiler built-in */ }),
+        (false) => ({ /* compiler built-in */ })
+    }
+
+    /// Attribute macro applied to a function to turn it into a unit test.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro test($item:item) { /* compiler built-in */ }
+
+    /// Attribute macro applied to a function to turn it into a benchmark test.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro bench($item:item) { /* compiler built-in */ }
+
+    /// An implementation detail of the `#[test]` and `#[bench]` macros.
+    #[unstable(feature = "custom_test_frameworks", issue = "50297",
+               reason = "custom test frameworks are an unstable feature")]
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    pub macro test_case($item:item) { /* compiler built-in */ }
+
+    /// Derive macro generating an impl of the trait `Clone`.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics, derive_clone_copy)]
+    pub macro Clone($item:item) { /* compiler built-in */ }
+
+    /// Derive macro generating an impl of the trait `Copy`.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics, derive_clone_copy)]
+    pub macro Copy($item:item) { /* compiler built-in */ }
+
+    /// Derive macro generating an impl of the trait `Debug`.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics)]
+    pub macro Debug($item:item) { /* compiler built-in */ }
+
+    /// Unstable implementation detail of the `rustc` compiler, do not use.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[rustc_deprecated(
+        since = "1.0.0",
+        reason = "derive(Decodable) is deprecated in favor of derive(RustcDecodable)",
+        suggestion = "RustcDecodable",
+    )]
+    #[allow_internal_unstable(core_intrinsics, libstd_sys_internals)]
+    pub macro Decodable($item:item) { /* compiler built-in */ }
+
+    /// Derive macro generating an impl of the trait `Default`.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics)]
+    pub macro Default($item:item) { /* compiler built-in */ }
+
+    /// Unstable implementation detail of the `rustc` compiler, do not use.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[rustc_deprecated(
+        since = "1.0.0",
+        reason = "derive(Encodable) is deprecated in favor of derive(RustcEncodable)",
+        suggestion = "RustcEncodable",
+    )]
+    #[allow_internal_unstable(core_intrinsics)]
+    pub macro Encodable($item:item) { /* compiler built-in */ }
+
+    /// Derive macro generating an impl of the trait `Eq`.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics, derive_eq)]
+    pub macro Eq($item:item) { /* compiler built-in */ }
+
+    /// Derive macro generating an impl of the trait `Hash`.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics)]
+    pub macro Hash($item:item) { /* compiler built-in */ }
+
+    /// Derive macro generating an impl of the trait `Ord`.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics)]
+    pub macro Ord($item:item) { /* compiler built-in */ }
+
+    /// Derive macro generating an impl of the trait `PartialEq`.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics)]
+    pub macro PartialEq($item:item) { /* compiler built-in */ }
+
+    /// Derive macro generating an impl of the trait `PartialOrd`.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics)]
+    pub macro PartialOrd($item:item) { /* compiler built-in */ }
+
+    /// Unstable implementation detail of the `rustc` compiler, do not use.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics, libstd_sys_internals)]
+    pub macro RustcDecodable($item:item) { /* compiler built-in */ }
+
+    /// Unstable implementation detail of the `rustc` compiler, do not use.
+    #[rustc_builtin_macro]
+    #[rustc_macro_transparency = "semitransparent"]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[allow_internal_unstable(core_intrinsics)]
+    pub macro RustcEncodable($item:item) { /* compiler built-in */ }
 }
diff --git a/src/libcore/unicode/unicode.py b/src/libcore/unicode/unicode.py
index ae356c3ff44..a0539cd9ca9 100755
--- a/src/libcore/unicode/unicode.py
+++ b/src/libcore/unicode/unicode.py
@@ -1,147 +1,350 @@
 #!/usr/bin/env python
 
-# This script uses the following Unicode tables:
-# - DerivedCoreProperties.txt
-# - DerivedNormalizationProps.txt
-# - EastAsianWidth.txt
-# - auxiliary/GraphemeBreakProperty.txt
-# - PropList.txt
-# - ReadMe.txt
-# - Scripts.txt
-# - UnicodeData.txt
-#
-# Since this should not require frequent updates, we just store this
-# out-of-line and check the tables.rs file into git.
+"""
+Regenerate Unicode tables (tables.rs).
+"""
 
-import fileinput, re, os, sys, operator, math, datetime
+# This script uses the Unicode tables as defined
+# in the UnicodeFiles class.
 
-# The directory in which this file resides.
-fdir = os.path.dirname(os.path.realpath(__file__)) + "/"
+# Since this should not require frequent updates, we just store this
+# out-of-line and check the tables.rs file into git.
 
-preamble = '''
+# Note that the "curl" program is required for operation.
+# This script is compatible with Python 2.7 and 3.x.
+
+import argparse
+import datetime
+import fileinput
+import itertools
+import os
+import re
+import textwrap
+import subprocess
+
+from collections import defaultdict, namedtuple
+
+try:
+    # Python 3
+    from itertools import zip_longest
+    from io import StringIO
+except ImportError:
+    # Python 2 compatibility
+    zip_longest = itertools.izip_longest
+    from StringIO import StringIO
+
+try:
+    # Completely optional type hinting
+    # (Python 2 compatible using comments,
+    # see: https://mypy.readthedocs.io/en/latest/python2.html)
+    # This is very helpful in typing-aware IDE like PyCharm.
+    from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple
+except ImportError:
+    pass
+
+
+# We don't use enum.Enum because of Python 2.7 compatibility.
+class UnicodeFiles(object):
+    # ReadMe does not contain any Unicode data, we
+    # only use it to extract versions.
+    README = "ReadMe.txt"
+
+    DERIVED_CORE_PROPERTIES = "DerivedCoreProperties.txt"
+    DERIVED_NORMALIZATION_PROPS = "DerivedNormalizationProps.txt"
+    PROPS = "PropList.txt"
+    SCRIPTS = "Scripts.txt"
+    SPECIAL_CASING = "SpecialCasing.txt"
+    UNICODE_DATA = "UnicodeData.txt"
+
+
+# The order doesn't really matter (Python < 3.6 won't preserve it),
+# we only want to aggregate all the file names.
+ALL_UNICODE_FILES = tuple(
+    value for name, value in UnicodeFiles.__dict__.items()
+    if not name.startswith("_")
+)
+
+assert len(ALL_UNICODE_FILES) == 7, "Unexpected number of unicode files"
+
+# The directory this file is located in.
+THIS_DIR = os.path.dirname(os.path.realpath(__file__))
+
+# Where to download the Unicode data.  The downloaded files
+# will be placed in sub-directories named after Unicode version.
+FETCH_DIR = os.path.join(THIS_DIR, "downloaded")
+
+FETCH_URL_LATEST = "ftp://ftp.unicode.org/Public/UNIDATA/{filename}"
+FETCH_URL_VERSION = "ftp://ftp.unicode.org/Public/{version}/ucd/{filename}"
+
+PREAMBLE = """\
 // NOTE: The following code was generated by "./unicode.py", do not edit directly
 
 #![allow(missing_docs, non_upper_case_globals, non_snake_case)]
 
 use unicode::version::UnicodeVersion;
 use unicode::bool_trie::{{BoolTrie, SmallBoolTrie}};
-'''.format(year = datetime.datetime.now().year)
+""".format(year=datetime.datetime.now().year)
 
 # Mapping taken from Table 12 from:
 # http://www.unicode.org/reports/tr44/#General_Category_Values
-expanded_categories = {
-    'Lu': ['LC', 'L'], 'Ll': ['LC', 'L'], 'Lt': ['LC', 'L'],
-    'Lm': ['L'], 'Lo': ['L'],
-    'Mn': ['M'], 'Mc': ['M'], 'Me': ['M'],
-    'Nd': ['N'], 'Nl': ['N'], 'No': ['N'],
-    'Pc': ['P'], 'Pd': ['P'], 'Ps': ['P'], 'Pe': ['P'],
-    'Pi': ['P'], 'Pf': ['P'], 'Po': ['P'],
-    'Sm': ['S'], 'Sc': ['S'], 'Sk': ['S'], 'So': ['S'],
-    'Zs': ['Z'], 'Zl': ['Z'], 'Zp': ['Z'],
-    'Cc': ['C'], 'Cf': ['C'], 'Cs': ['C'], 'Co': ['C'], 'Cn': ['C'],
+EXPANDED_CATEGORIES = {
+    "Lu": ["LC", "L"], "Ll": ["LC", "L"], "Lt": ["LC", "L"],
+    "Lm": ["L"], "Lo": ["L"],
+    "Mn": ["M"], "Mc": ["M"], "Me": ["M"],
+    "Nd": ["N"], "Nl": ["N"], "No": ["N"],
+    "Pc": ["P"], "Pd": ["P"], "Ps": ["P"], "Pe": ["P"],
+    "Pi": ["P"], "Pf": ["P"], "Po": ["P"],
+    "Sm": ["S"], "Sc": ["S"], "Sk": ["S"], "So": ["S"],
+    "Zs": ["Z"], "Zl": ["Z"], "Zp": ["Z"],
+    "Cc": ["C"], "Cf": ["C"], "Cs": ["C"], "Co": ["C"], "Cn": ["C"],
 }
 
-# these are the surrogate codepoints, which are not valid rust characters
-surrogate_codepoints = (0xd800, 0xdfff)
+# This is the (inclusive) range of surrogate codepoints.
+# These are not valid Rust characters.
+SURROGATE_CODEPOINTS_RANGE = (0xd800, 0xdfff)
+
+UnicodeData = namedtuple(
+    "UnicodeData", (
+        # Conversions:
+        "to_upper", "to_lower", "to_title",
+
+        # Decompositions: canonical decompositions, compatibility decomp
+        "canon_decomp", "compat_decomp",
+
+        # Grouped: general categories and combining characters
+        "general_categories", "combines",
+    )
+)
+
+UnicodeVersion = namedtuple(
+    "UnicodeVersion", ("major", "minor", "micro", "as_str")
+)
+
+
+def fetch_files(version=None):
+    # type: (str) -> UnicodeVersion
+    """
+    Fetch all the Unicode files from unicode.org.
+
+    This will use cached files (stored in `FETCH_DIR`) if they exist,
+    creating them if they don't.  In any case, the Unicode version
+    is always returned.
+
+    :param version: The desired Unicode version, as string.
+        (If None, defaults to latest final release available,
+         querying the unicode.org service).
+    """
+    have_version = check_stored_version(version)
+    if have_version:
+        return have_version
+
+    if version:
+        # Check if the desired version exists on the server.
+        get_fetch_url = lambda name: FETCH_URL_VERSION.format(version=version, filename=name)
+    else:
+        # Extract the latest version.
+        get_fetch_url = lambda name: FETCH_URL_LATEST.format(filename=name)
+
+    readme_url = get_fetch_url(UnicodeFiles.README)
+
+    print("Fetching: {}".format(readme_url))
+    readme_content = subprocess.check_output(("curl", readme_url))
+
+    unicode_version = parse_readme_unicode_version(
+        readme_content.decode("utf8")
+    )
+
+    download_dir = get_unicode_dir(unicode_version)
+    if not os.path.exists(download_dir):
+        # For 2.7 compat, we don't use `exist_ok=True`.
+        os.makedirs(download_dir)
+
+    for filename in ALL_UNICODE_FILES:
+        file_path = get_unicode_file_path(unicode_version, filename)
+
+        if os.path.exists(file_path):
+            # Assume file on the server didn't change if it's been saved before.
+            continue
+
+        if filename == UnicodeFiles.README:
+            with open(file_path, "wb") as fd:
+                fd.write(readme_content)
+        else:
+            url = get_fetch_url(filename)
+            print("Fetching: {}".format(url))
+            subprocess.check_call(("curl", "-o", file_path, url))
+
+    return unicode_version
+
+
+def check_stored_version(version):
+    # type: (Optional[str]) -> Optional[UnicodeVersion]
+    """
+    Given desired Unicode version, return the version
+    if stored files are all present, and `None` otherwise.
+    """
+    if not version:
+        # If no desired version specified, we should check what's the latest
+        # version, skipping stored version checks.
+        return None
+
+    fetch_dir = os.path.join(FETCH_DIR, version)
 
-def fetch(f):
-    path = fdir + os.path.basename(f)
-    if not os.path.exists(path):
-        os.system("curl -o {0}{1} ftp://ftp.unicode.org/Public/UNIDATA/{1}".format(fdir, f))
+    for filename in ALL_UNICODE_FILES:
+        file_path = os.path.join(fetch_dir, filename)
+
+        if not os.path.exists(file_path):
+            return None
+
+    with open(os.path.join(fetch_dir, UnicodeFiles.README)) as fd:
+        return parse_readme_unicode_version(fd.read())
+
+
+def parse_readme_unicode_version(readme_content):
+    # type: (str) -> UnicodeVersion
+    """
+    Parse the Unicode version contained in their `ReadMe.txt` file.
+    """
+    # "Raw string" is necessary for \d not being treated as escape char
+    # (for the sake of compat with future Python versions).
+    # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior
+    pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode"
+    groups = re.search(pattern, readme_content).groups()
+
+    return UnicodeVersion(*map(int, groups), as_str=".".join(groups))
+
+
+def get_unicode_dir(unicode_version):
+    # type: (UnicodeVersion) -> str
+    """
+    Indicate in which parent dir the Unicode data files should be stored.
+
+    This returns a full, absolute path.
+    """
+    return os.path.join(FETCH_DIR, unicode_version.as_str)
+
+
+def get_unicode_file_path(unicode_version, filename):
+    # type: (UnicodeVersion, str) -> str
+    """
+    Indicate where the Unicode data file should be stored.
+    """
+    return os.path.join(get_unicode_dir(unicode_version), filename)
 
-    if not os.path.exists(path):
-        sys.stderr.write("cannot load %s" % f)
-        exit(1)
 
 def is_surrogate(n):
-    return surrogate_codepoints[0] <= n <= surrogate_codepoints[1]
-
-def load_unicode_data(f):
-    fetch(f)
-    gencats = {}
-    to_lower = {}
-    to_upper = {}
-    to_title = {}
-    combines = {}
-    canon_decomp = {}
-    compat_decomp = {}
-
-    udict = {}
+    # type: (int) -> bool
+    """
+    Tell if given codepoint is a surrogate (not a valid Rust character).
+    """
+    return SURROGATE_CODEPOINTS_RANGE[0] <= n <= SURROGATE_CODEPOINTS_RANGE[1]
+
+
+def load_unicode_data(file_path):
+    # type: (str) -> UnicodeData
+    """
+    Load main Unicode data.
+    """
+    # Conversions
+    to_lower = {}   # type: Dict[int, Tuple[int, int, int]]
+    to_upper = {}   # type: Dict[int, Tuple[int, int, int]]
+    to_title = {}   # type: Dict[int, Tuple[int, int, int]]
+
+    # Decompositions
+    compat_decomp = {}   # type: Dict[int, List[int]]
+    canon_decomp = {}    # type: Dict[int, List[int]]
+
+    # Combining characters
+    # FIXME: combines are not used
+    combines = defaultdict(set)   # type: Dict[str, Set[int]]
+
+    # Categories
+    general_categories = defaultdict(set)   # type: Dict[str, Set[int]]
+    category_assigned_codepoints = set()    # type: Set[int]
+
+    all_codepoints = {}
+
     range_start = -1
-    for line in fileinput.input(fdir + f):
-        data = line.split(';')
+
+    for line in fileinput.input(file_path):
+        data = line.split(";")
         if len(data) != 15:
             continue
-        cp = int(data[0], 16)
-        if is_surrogate(cp):
+        codepoint = int(data[0], 16)
+        if is_surrogate(codepoint):
             continue
         if range_start >= 0:
-            for i in range(range_start, cp):
-                udict[i] = data
+            for i in range(range_start, codepoint):
+                all_codepoints[i] = data
             range_start = -1
         if data[1].endswith(", First>"):
-            range_start = cp
+            range_start = codepoint
             continue
-        udict[cp] = data
+        all_codepoints[codepoint] = data
 
-    for code in udict:
+    for code, data in all_codepoints.items():
         (code_org, name, gencat, combine, bidi,
          decomp, deci, digit, num, mirror,
-         old, iso, upcase, lowcase, titlecase) = udict[code]
+         old, iso, upcase, lowcase, titlecase) = data
+
+        # Generate char to char direct common and simple conversions:
 
-        # generate char to char direct common and simple conversions
-        # uppercase to lowercase
+        # Uppercase to lowercase
         if lowcase != "" and code_org != lowcase:
             to_lower[code] = (int(lowcase, 16), 0, 0)
 
-        # lowercase to uppercase
+        # Lowercase to uppercase
         if upcase != "" and code_org != upcase:
             to_upper[code] = (int(upcase, 16), 0, 0)
 
-        # title case
+        # Title case
         if titlecase.strip() != "" and code_org != titlecase:
             to_title[code] = (int(titlecase, 16), 0, 0)
 
-        # store decomposition, if given
-        if decomp != "":
-            if decomp.startswith('<'):
-                seq = []
-                for i in decomp.split()[1:]:
-                    seq.append(int(i, 16))
-                compat_decomp[code] = seq
+        # Store decomposition, if given
+        if decomp:
+            decompositions = decomp.split()[1:]
+            decomp_code_points = [int(i, 16) for i in decompositions]
+
+            if decomp.startswith("<"):
+                # Compatibility decomposition
+                compat_decomp[code] = decomp_code_points
             else:
-                seq = []
-                for i in decomp.split():
-                    seq.append(int(i, 16))
-                canon_decomp[code] = seq
-
-        # place letter in categories as appropriate
-        for cat in [gencat, "Assigned"] + expanded_categories.get(gencat, []):
-            if cat not in gencats:
-                gencats[cat] = []
-            gencats[cat].append(code)
-
-        # record combining class, if any
+                # Canonical decomposition
+                canon_decomp[code] = decomp_code_points
+
+        # Place letter in categories as appropriate.
+        for cat in itertools.chain((gencat, ), EXPANDED_CATEGORIES.get(gencat, [])):
+            general_categories[cat].add(code)
+            category_assigned_codepoints.add(code)
+
+        # Record combining class, if any.
         if combine != "0":
-            if combine not in combines:
-                combines[combine] = []
-            combines[combine].append(code)
-
-    # generate Not_Assigned from Assigned
-    gencats["Cn"] = gen_unassigned(gencats["Assigned"])
-    # Assigned is not a real category
-    del(gencats["Assigned"])
+            combines[combine].add(code)
+
+    # Generate Not_Assigned from Assigned.
+    general_categories["Cn"] = get_unassigned_codepoints(category_assigned_codepoints)
+
     # Other contains Not_Assigned
-    gencats["C"].extend(gencats["Cn"])
-    gencats = group_cats(gencats)
-    combines = to_combines(group_cats(combines))
+    general_categories["C"].update(general_categories["Cn"])
+
+    grouped_categories = group_categories(general_categories)
 
-    return (canon_decomp, compat_decomp, gencats, combines, to_upper, to_lower, to_title)
+    # FIXME: combines are not used
+    return UnicodeData(
+        to_lower=to_lower, to_upper=to_upper, to_title=to_title,
+        compat_decomp=compat_decomp, canon_decomp=canon_decomp,
+        general_categories=grouped_categories, combines=combines,
+    )
 
-def load_special_casing(f, to_upper, to_lower, to_title):
-    fetch(f)
-    for line in fileinput.input(fdir + f):
-        data = line.split('#')[0].split(';')
+
+def load_special_casing(file_path, unicode_data):
+    # type: (str, UnicodeData) -> None
+    """
+    Load special casing data and enrich given Unicode data.
+    """
+    for line in fileinput.input(file_path):
+        data = line.split("#")[0].split(";")
         if len(data) == 5:
             code, lower, title, upper, _comment = data
         elif len(data) == 6:
@@ -155,243 +358,399 @@ def load_special_casing(f, to_upper, to_lower, to_title):
         title = title.strip()
         upper = upper.strip()
         key = int(code, 16)
-        for (map_, values) in [(to_lower, lower), (to_upper, upper), (to_title, title)]:
+        for (map_, values) in ((unicode_data.to_lower, lower),
+                               (unicode_data.to_upper, upper),
+                               (unicode_data.to_title, title)):
             if values != code:
-                values = [int(i, 16) for i in values.split()]
-                for _ in range(len(values), 3):
-                    values.append(0)
-                assert len(values) == 3
-                map_[key] = values
-
-def group_cats(cats):
-    cats_out = {}
-    for cat in cats:
-        cats_out[cat] = group_cat(cats[cat])
-    return cats_out
-
-def group_cat(cat):
-    cat_out = []
-    letters = sorted(set(cat))
-    cur_start = letters.pop(0)
-    cur_end = cur_start
-    for letter in letters:
-        assert letter > cur_end, \
-            "cur_end: %s, letter: %s" % (hex(cur_end), hex(letter))
-        if letter == cur_end + 1:
-            cur_end = letter
-        else:
-            cat_out.append((cur_start, cur_end))
-            cur_start = cur_end = letter
-    cat_out.append((cur_start, cur_end))
-    return cat_out
-
-def ungroup_cat(cat):
-    cat_out = []
-    for (lo, hi) in cat:
-        while lo <= hi:
-            cat_out.append(lo)
-            lo += 1
-    return cat_out
-
-def gen_unassigned(assigned):
-    assigned = set(assigned)
-    return ([i for i in range(0, 0xd800) if i not in assigned] +
-            [i for i in range(0xe000, 0x110000) if i not in assigned])
-
-def to_combines(combs):
-    combs_out = []
-    for comb in combs:
-        for (lo, hi) in combs[comb]:
-            combs_out.append((lo, hi, comb))
-    combs_out.sort(key=lambda comb: comb[0])
-    return combs_out
-
-def format_table_content(f, content, indent):
-    line = " "*indent
+                split = values.split()
+
+                codepoints = list(itertools.chain(
+                    (int(i, 16) for i in split),
+                    (0 for _ in range(len(split), 3))
+                ))
+
+                assert len(codepoints) == 3
+                map_[key] = codepoints
+
+
+def group_categories(mapping):
+    # type: (Dict[Any, Iterable[int]]) -> Dict[str, List[Tuple[int, int]]]
+    """
+    Group codepoints mapped in "categories".
+    """
+    return {category: group_codepoints(codepoints)
+            for category, codepoints in mapping.items()}
+
+
+def group_codepoints(codepoints):
+    # type: (Iterable[int]) -> List[Tuple[int, int]]
+    """
+    Group integral values into continuous, disjoint value ranges.
+
+    Performs value deduplication.
+
+    :return: sorted list of pairs denoting start and end of codepoint
+        group values, both ends inclusive.
+
+    >>> group_codepoints([1, 2, 10, 11, 12, 3, 4])
+    [(1, 4), (10, 12)]
+    >>> group_codepoints([1])
+    [(1, 1)]
+    >>> group_codepoints([1, 5, 6])
+    [(1, 1), (5, 6)]
+    >>> group_codepoints([])
+    []
+    """
+    sorted_codes = sorted(set(codepoints))
+    result = []     # type: List[Tuple[int, int]]
+
+    if not sorted_codes:
+        return result
+
+    next_codes = sorted_codes[1:]
+    start_code = sorted_codes[0]
+
+    for code, next_code in zip_longest(sorted_codes, next_codes, fillvalue=None):
+        if next_code is None or next_code - code != 1:
+            result.append((start_code, code))
+            start_code = next_code
+
+    return result
+
+
+def ungroup_codepoints(codepoint_pairs):
+    # type: (Iterable[Tuple[int, int]]) -> List[int]
+    """
+    The inverse of group_codepoints -- produce a flat list of values
+    from value range pairs.
+
+    >>> ungroup_codepoints([(1, 4), (10, 12)])
+    [1, 2, 3, 4, 10, 11, 12]
+    >>> ungroup_codepoints([(1, 1), (5, 6)])
+    [1, 5, 6]
+    >>> ungroup_codepoints(group_codepoints([1, 2, 7, 8]))
+    [1, 2, 7, 8]
+    >>> ungroup_codepoints([])
+    []
+    """
+    return list(itertools.chain.from_iterable(
+        range(lo, hi + 1) for lo, hi in codepoint_pairs
+    ))
+
+
+def get_unassigned_codepoints(assigned_codepoints):
+    # type: (Set[int]) -> Set[int]
+    """
+    Given a set of "assigned" codepoints, return a set
+    of these that are not in assigned and not surrogate.
+    """
+    return {i for i in range(0, 0x110000)
+            if i not in assigned_codepoints and not is_surrogate(i)}
+
+
+def generate_table_lines(items, indent, wrap=98):
+    # type: (Iterable[str], int, int) -> Iterator[str]
+    """
+    Given table items, generate wrapped lines of text with comma-separated items.
+
+    This is a generator function.
+
+    :param wrap: soft wrap limit (characters per line), integer.
+    """
+    line = " " * indent
     first = True
-    for chunk in content.split(","):
-        if len(line) + len(chunk) < 98:
+    for item in items:
+        if len(line) + len(item) < wrap:
             if first:
-                line += chunk
+                line += item
             else:
-                line += ", " + chunk
+                line += ", " + item
             first = False
         else:
-            f.write(line + ",\n")
-            line = " "*indent + chunk
-    f.write(line)
-
-def load_properties(f, interestingprops):
-    fetch(f)
-    props = {}
-    re1 = re.compile("^ *([0-9A-F]+) *; *(\w+)")
-    re2 = re.compile("^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *(\w+)")
-
-    for line in fileinput.input(fdir + os.path.basename(f)):
-        prop = None
-        d_lo = 0
-        d_hi = 0
-        m = re1.match(line)
-        if m:
-            d_lo = m.group(1)
-            d_hi = m.group(1)
-            prop = m.group(2)
-        else:
-            m = re2.match(line)
-            if m:
-                d_lo = m.group(1)
-                d_hi = m.group(2)
-                prop = m.group(3)
+            yield line + ",\n"
+            line = " " * indent + item
+
+    yield line
+
+
+def load_properties(file_path, interesting_props):
+    # type: (str, Iterable[str]) -> Dict[str, List[Tuple[int, int]]]
+    """
+    Load properties data and return in grouped form.
+    """
+    props = defaultdict(list)   # type: Dict[str, List[Tuple[int, int]]]
+    # "Raw string" is necessary for `\.` and `\w` not to be treated as escape chars
+    # (for the sake of compat with future Python versions).
+    # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior
+    re1 = re.compile(r"^ *([0-9A-F]+) *; *(\w+)")
+    re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *(\w+)")
+
+    for line in fileinput.input(file_path):
+        match = re1.match(line) or re2.match(line)
+        if match:
+            groups = match.groups()
+
+            if len(groups) == 2:
+                # `re1` matched (2 groups).
+                d_lo, prop = groups
+                d_hi = d_lo
             else:
-                continue
-        if interestingprops and prop not in interestingprops:
+                d_lo, d_hi, prop = groups
+        else:
+            continue
+
+        if interesting_props and prop not in interesting_props:
             continue
-        d_lo = int(d_lo, 16)
-        d_hi = int(d_hi, 16)
-        if prop not in props:
-            props[prop] = []
-        props[prop].append((d_lo, d_hi))
 
-    # optimize if possible
+        lo_value = int(d_lo, 16)
+        hi_value = int(d_hi, 16)
+
+        props[prop].append((lo_value, hi_value))
+
+    # Optimize if possible.
     for prop in props:
-        props[prop] = group_cat(ungroup_cat(props[prop]))
+        props[prop] = group_codepoints(ungroup_codepoints(props[prop]))
 
     return props
 
-def escape_char(c):
-    return "'\\u{%x}'" % c if c != 0 else "'\\0'"
 
-def emit_table(f, name, t_data, t_type = "&[(char, char)]", is_pub=True,
-        pfun=lambda x: "(%s,%s)" % (escape_char(x[0]), escape_char(x[1]))):
+def escape_char(c):
+    # type: (int) -> str
+    r"""
+    Escape a codepoint for use as Rust char literal.
+
+    Outputs are OK to use as Rust source code as char literals
+    and they also include necessary quotes.
+
+    >>> escape_char(97)
+    "'\\u{61}'"
+    >>> escape_char(0)
+    "'\\0'"
+    """
+    return r"'\u{%x}'" % c if c != 0 else r"'\0'"
+
+
+def format_char_pair(pair):
+    # type: (Tuple[int, int]) -> str
+    """
+    Format a pair of two Rust chars.
+    """
+    return "(%s,%s)" % (escape_char(pair[0]), escape_char(pair[1]))
+
+
+def generate_table(
+    name,   # type: str
+    items,  # type: List[Tuple[int, int]]
+    decl_type="&[(char, char)]",    # type: str
+    is_pub=True,                    # type: bool
+    format_item=format_char_pair,   # type: Callable[[Tuple[int, int]], str]
+):
+    # type: (...) -> Iterator[str]
+    """
+    Generate a nicely formatted Rust constant "table" array.
+
+    This generates actual Rust code.
+    """
     pub_string = ""
     if is_pub:
         pub_string = "pub "
-    f.write("    %sconst %s: %s = &[\n" % (pub_string, name, t_type))
-    data = ""
+
+    yield "    %sconst %s: %s = &[\n" % (pub_string, name, decl_type)
+
+    data = []
     first = True
-    for dat in t_data:
+    for item in items:
         if not first:
-            data += ","
+            data.append(",")
         first = False
-        data += pfun(dat)
-    format_table_content(f, data, 8)
-    f.write("\n    ];\n\n")
+        data.extend(format_item(item))
 
-def compute_trie(rawdata, chunksize):
+    for table_line in generate_table_lines("".join(data).split(","), 8):
+        yield table_line
+
+    yield "\n    ];\n\n"
+
+
+def compute_trie(raw_data, chunk_size):
+    # type: (List[int], int) -> Tuple[List[int], List[int]]
+    """
+    Compute postfix-compressed trie.
+
+    See: bool_trie.rs for more details.
+
+    >>> compute_trie([1, 2, 3, 1, 2, 3, 4, 5, 6], 3)
+    ([0, 0, 1], [1, 2, 3, 4, 5, 6])
+    >>> compute_trie([1, 2, 3, 1, 2, 4, 4, 5, 6], 3)
+    ([0, 1, 2], [1, 2, 3, 1, 2, 4, 4, 5, 6])
+    """
     root = []
-    childmap = {}
+    childmap = {}       # type: Dict[Tuple[int, ...], int]
     child_data = []
-    for i in range(len(rawdata) // chunksize):
-        data = rawdata[i * chunksize: (i + 1) * chunksize]
-        child = '|'.join(map(str, data))
+
+    assert len(raw_data) % chunk_size == 0, "Chunks must be equally sized"
+
+    for i in range(len(raw_data) // chunk_size):
+        data = raw_data[i * chunk_size : (i + 1) * chunk_size]
+
+        # Postfix compression of child nodes (data chunks)
+        # (identical child nodes are shared).
+
+        # Make a tuple out of the list so it's hashable.
+        child = tuple(data)
         if child not in childmap:
             childmap[child] = len(childmap)
             child_data.extend(data)
+
         root.append(childmap[child])
-    return (root, child_data)
 
-def emit_bool_trie(f, name, t_data, is_pub=True):
-    CHUNK = 64
+    return root, child_data
+
+
+def generate_bool_trie(name, codepoint_ranges, is_pub=True):
+    # type: (str, List[Tuple[int, int]], bool) -> Iterator[str]
+    """
+    Generate Rust code for BoolTrie struct.
+
+    This yields string fragments that should be joined to produce
+    the final string.
+
+    See: `bool_trie.rs`.
+    """
+    chunk_size = 64
     rawdata = [False] * 0x110000
-    for (lo, hi) in t_data:
+    for (lo, hi) in codepoint_ranges:
         for cp in range(lo, hi + 1):
             rawdata[cp] = True
 
-    # convert to bitmap chunks of 64 bits each
+    # Convert to bitmap chunks of `chunk_size` bits each.
     chunks = []
-    for i in range(0x110000 // CHUNK):
+    for i in range(0x110000 // chunk_size):
         chunk = 0
-        for j in range(64):
-            if rawdata[i * 64 + j]:
+        for j in range(chunk_size):
+            if rawdata[i * chunk_size + j]:
                 chunk |= 1 << j
         chunks.append(chunk)
 
     pub_string = ""
     if is_pub:
         pub_string = "pub "
-    f.write("    %sconst %s: &super::BoolTrie = &super::BoolTrie {\n" % (pub_string, name))
-    f.write("        r1: [\n")
-    data = ','.join('0x%016x' % chunk for chunk in chunks[0:0x800 // CHUNK])
-    format_table_content(f, data, 12)
-    f.write("\n        ],\n")
+    yield "    %sconst %s: &super::BoolTrie = &super::BoolTrie {\n" % (pub_string, name)
+    yield "        r1: [\n"
+    data = ("0x%016x" % chunk for chunk in chunks[:0x800 // chunk_size])
+    for fragment in generate_table_lines(data, 12):
+        yield fragment
+    yield "\n        ],\n"
 
     # 0x800..0x10000 trie
-    (r2, r3) = compute_trie(chunks[0x800 // CHUNK : 0x10000 // CHUNK], 64 // CHUNK)
-    f.write("        r2: [\n")
-    data = ','.join(str(node) for node in r2)
-    format_table_content(f, data, 12)
-    f.write("\n        ],\n")
-    f.write("        r3: &[\n")
-    data = ','.join('0x%016x' % chunk for chunk in r3)
-    format_table_content(f, data, 12)
-    f.write("\n        ],\n")
+    (r2, r3) = compute_trie(chunks[0x800 // chunk_size : 0x10000 // chunk_size], 64 // chunk_size)
+    yield "        r2: [\n"
+    data = map(str, r2)
+    for fragment in generate_table_lines(data, 12):
+        yield fragment
+    yield "\n        ],\n"
+
+    yield "        r3: &[\n"
+    data = ("0x%016x" % node for node in r3)
+    for fragment in generate_table_lines(data, 12):
+        yield fragment
+    yield "\n        ],\n"
 
     # 0x10000..0x110000 trie
-    (mid, r6) = compute_trie(chunks[0x10000 // CHUNK : 0x110000 // CHUNK], 64 // CHUNK)
+    (mid, r6) = compute_trie(chunks[0x10000 // chunk_size : 0x110000 // chunk_size],
+                             64 // chunk_size)
     (r4, r5) = compute_trie(mid, 64)
-    f.write("        r4: [\n")
-    data = ','.join(str(node) for node in r4)
-    format_table_content(f, data, 12)
-    f.write("\n        ],\n")
-    f.write("        r5: &[\n")
-    data = ','.join(str(node) for node in r5)
-    format_table_content(f, data, 12)
-    f.write("\n        ],\n")
-    f.write("        r6: &[\n")
-    data = ','.join('0x%016x' % chunk for chunk in r6)
-    format_table_content(f, data, 12)
-    f.write("\n        ],\n")
-
-    f.write("    };\n\n")
-
-def emit_small_bool_trie(f, name, t_data, is_pub=True):
-    last_chunk = max(hi // 64 for (lo, hi) in t_data)
+
+    yield "        r4: [\n"
+    data = map(str, r4)
+    for fragment in generate_table_lines(data, 12):
+        yield fragment
+    yield "\n        ],\n"
+
+    yield "        r5: &[\n"
+    data = map(str, r5)
+    for fragment in generate_table_lines(data, 12):
+        yield fragment
+    yield "\n        ],\n"
+
+    yield "        r6: &[\n"
+    data = ("0x%016x" % node for node in r6)
+    for fragment in generate_table_lines(data, 12):
+        yield fragment
+    yield "\n        ],\n"
+
+    yield "    };\n\n"
+
+
+def generate_small_bool_trie(name, codepoint_ranges, is_pub=True):
+    # type: (str, List[Tuple[int, int]], bool) -> Iterator[str]
+    """
+    Generate Rust code for `SmallBoolTrie` struct.
+
+    See: `bool_trie.rs`.
+    """
+    last_chunk = max(hi // 64 for (lo, hi) in codepoint_ranges)
     n_chunks = last_chunk + 1
     chunks = [0] * n_chunks
-    for (lo, hi) in t_data:
+    for (lo, hi) in codepoint_ranges:
         for cp in range(lo, hi + 1):
-            if cp // 64 >= len(chunks):
-                print(cp, cp // 64, len(chunks), lo, hi)
+            assert cp // 64 < len(chunks)
             chunks[cp // 64] |= 1 << (cp & 63)
 
     pub_string = ""
     if is_pub:
         pub_string = "pub "
-    f.write("    %sconst %s: &super::SmallBoolTrie = &super::SmallBoolTrie {\n"
-            % (pub_string, name))
+
+    yield ("    %sconst %s: &super::SmallBoolTrie = &super::SmallBoolTrie {\n"
+           % (pub_string, name))
 
     (r1, r2) = compute_trie(chunks, 1)
 
-    f.write("        r1: &[\n")
-    data = ','.join(str(node) for node in r1)
-    format_table_content(f, data, 12)
-    f.write("\n        ],\n")
-
-    f.write("        r2: &[\n")
-    data = ','.join('0x%016x' % node for node in r2)
-    format_table_content(f, data, 12)
-    f.write("\n        ],\n")
-
-    f.write("    };\n\n")
-
-def emit_property_module(f, mod, tbl, emit):
-    f.write("pub mod %s {\n" % mod)
-    for cat in sorted(emit):
-        if cat in ["Cc", "White_Space", "Pattern_White_Space"]:
-            emit_small_bool_trie(f, "%s_table" % cat, tbl[cat])
-            f.write("    pub fn %s(c: char) -> bool {\n" % cat)
-            f.write("        %s_table.lookup(c)\n" % cat)
-            f.write("    }\n\n")
+    yield "        r1: &[\n"
+    data = (str(node) for node in r1)
+    for fragment in generate_table_lines(data, 12):
+        yield fragment
+    yield "\n        ],\n"
+
+    yield "        r2: &[\n"
+    data = ("0x%016x" % node for node in r2)
+    for fragment in generate_table_lines(data, 12):
+        yield fragment
+    yield "\n        ],\n"
+
+    yield "    };\n\n"
+
+
+def generate_property_module(mod, grouped_categories, category_subset):
+    # type: (str, Dict[str, List[Tuple[int, int]]], Iterable[str]) -> Iterator[str]
+    """
+    Generate Rust code for module defining properties.
+    """
+
+    yield "pub mod %s {\n" % mod
+    for cat in sorted(category_subset):
+        if cat in ("Cc", "White_Space", "Pattern_White_Space"):
+            generator = generate_small_bool_trie("%s_table" % cat, grouped_categories[cat])
         else:
-            emit_bool_trie(f, "%s_table" % cat, tbl[cat])
-            f.write("    pub fn %s(c: char) -> bool {\n" % cat)
-            f.write("        %s_table.lookup(c)\n" % cat)
-            f.write("    }\n\n")
-    f.write("}\n\n")
-
-def emit_conversions_module(f, to_upper, to_lower, to_title):
-    f.write("pub mod conversions {")
-    f.write("""
+            generator = generate_bool_trie("%s_table" % cat, grouped_categories[cat])
+
+        for fragment in generator:
+            yield fragment
+
+        yield "    pub fn %s(c: char) -> bool {\n" % cat
+        yield "        %s_table.lookup(c)\n" % cat
+        yield "    }\n\n"
+
+    yield "}\n\n"
+
+
+def generate_conversions_module(unicode_data):
+    # type: (UnicodeData) -> Iterator[str]
+    """
+    Generate Rust code for module defining conversions.
+    """
+
+    yield "pub mod conversions {"
+    yield """
     pub fn to_lower(c: char) -> [char; 3] {
         match bsearch_case_table(c, to_lowercase_table) {
             None        => [c, '\\0', '\\0'],
@@ -408,80 +767,109 @@ def emit_conversions_module(f, to_upper, to_lower, to_title):
 
     fn bsearch_case_table(c: char, table: &[(char, [char; 3])]) -> Option<usize> {
         table.binary_search_by(|&(key, _)| key.cmp(&c)).ok()
-    }
+    }\n\n"""
+
+    decl_type = "&[(char, [char; 3])]"
+    format_conversion = lambda x: "({},[{},{},{}])".format(*(
+        escape_char(c) for c in (x[0], x[1][0], x[1][1], x[1][2])
+    ))
+
+    for fragment in generate_table(
+        name="to_lowercase_table",
+        items=sorted(unicode_data.to_lower.items(), key=lambda x: x[0]),
+        decl_type=decl_type,
+        is_pub=False,
+        format_item=format_conversion
+    ):
+        yield fragment
+
+    for fragment in generate_table(
+        name="to_uppercase_table",
+        items=sorted(unicode_data.to_upper.items(), key=lambda x: x[0]),
+        decl_type=decl_type,
+        is_pub=False,
+        format_item=format_conversion
+    ):
+        yield fragment
+
+    yield "}\n"
+
+
+def parse_args():
+    # type: () -> argparse.Namespace
+    """
+    Parse command line arguments.
+    """
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("-v", "--version", default=None, type=str,
+                        help="Unicode version to use (if not specified,"
+                             " defaults to latest release).")
+
+    return parser.parse_args()
+
+
+def main():
+    # type: () -> None
+    """
+    Script entry point.
+    """
+    args = parse_args()
+
+    unicode_version = fetch_files(args.version)
+    print("Using Unicode version: {}".format(unicode_version.as_str))
+
+    # All the writing happens entirely in memory, we only write to file
+    # once we have generated the file content (it's not very large, <1 MB).
+    buf = StringIO()
+    buf.write(PREAMBLE)
+
+    unicode_version_notice = textwrap.dedent("""
+    /// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of
+    /// `char` and `str` methods are based on.
+    #[unstable(feature = "unicode_version", issue = "49726")]
+    pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion {{
+        major: {version.major},
+        minor: {version.minor},
+        micro: {version.micro},
+        _priv: (),
+    }};
+    """).format(version=unicode_version)
+    buf.write(unicode_version_notice)
+
+    get_path = lambda f: get_unicode_file_path(unicode_version, f)
+
+    unicode_data = load_unicode_data(get_path(UnicodeFiles.UNICODE_DATA))
+    load_special_casing(get_path(UnicodeFiles.SPECIAL_CASING), unicode_data)
+
+    want_derived = {"XID_Start", "XID_Continue", "Alphabetic", "Lowercase", "Uppercase",
+                    "Cased", "Case_Ignorable", "Grapheme_Extend"}
+    derived = load_properties(get_path(UnicodeFiles.DERIVED_CORE_PROPERTIES), want_derived)
+
+    props = load_properties(get_path(UnicodeFiles.PROPS),
+                            {"White_Space", "Join_Control", "Noncharacter_Code_Point",
+                             "Pattern_White_Space"})
+
+    # Category tables
+    for (name, categories, category_subset) in (
+            ("general_category", unicode_data.general_categories, ["N", "Cc"]),
+            ("derived_property", derived, want_derived),
+            ("property", props, ["White_Space", "Pattern_White_Space"])
+    ):
+        for fragment in generate_property_module(name, categories, category_subset):
+            buf.write(fragment)
+
+    for fragment in generate_conversions_module(unicode_data):
+        buf.write(fragment)
+
+    tables_rs_path = os.path.join(THIS_DIR, "tables.rs")
+
+    # Actually write out the file content.
+    # Will overwrite the file if it exists.
+    with open(tables_rs_path, "w") as fd:
+        fd.write(buf.getvalue())
+
+    print("Regenerated tables.rs.")
 
-""")
-    t_type = "&[(char, [char; 3])]"
-    pfun = lambda x: "(%s,[%s,%s,%s])" % (
-        escape_char(x[0]), escape_char(x[1][0]), escape_char(x[1][1]), escape_char(x[1][2]))
-    emit_table(f, "to_lowercase_table",
-        sorted(to_lower.items(), key=operator.itemgetter(0)),
-        is_pub=False, t_type = t_type, pfun=pfun)
-    emit_table(f, "to_uppercase_table",
-        sorted(to_upper.items(), key=operator.itemgetter(0)),
-        is_pub=False, t_type = t_type, pfun=pfun)
-    f.write("}\n\n")
-
-def emit_norm_module(f, canon, compat, combine, norm_props):
-    canon_keys = sorted(canon.keys())
-
-    compat_keys = sorted(compat.keys())
-
-    canon_comp = {}
-    comp_exclusions = norm_props["Full_Composition_Exclusion"]
-    for char in canon_keys:
-        if any(lo <= char <= hi for lo, hi in comp_exclusions):
-            continue
-        decomp = canon[char]
-        if len(decomp) == 2:
-            if decomp[0] not in canon_comp:
-                canon_comp[decomp[0]] = []
-            canon_comp[decomp[0]].append( (decomp[1], char) )
-    canon_comp_keys = sorted(canon_comp.keys())
 
 if __name__ == "__main__":
-    r = fdir + "tables.rs"
-    if os.path.exists(r):
-        os.remove(r)
-    with open(r, "w") as rf:
-        # write the file's preamble
-        rf.write(preamble)
-
-        # download and parse all the data
-        fetch("ReadMe.txt")
-        with open(fdir + "ReadMe.txt") as readme:
-            pattern = "for Version (\d+)\.(\d+)\.(\d+) of the Unicode"
-            unicode_version = re.search(pattern, readme.read()).groups()
-        rf.write("""
-/// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of
-/// `char` and `str` methods are based on.
-#[unstable(feature = "unicode_version", issue = "49726")]
-pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion {
-    major: %s,
-    minor: %s,
-    micro: %s,
-    _priv: (),
-};
-""" % unicode_version)
-        (canon_decomp, compat_decomp, gencats, combines,
-                to_upper, to_lower, to_title) = load_unicode_data("UnicodeData.txt")
-        load_special_casing("SpecialCasing.txt", to_upper, to_lower, to_title)
-        want_derived = ["XID_Start", "XID_Continue", "Alphabetic", "Lowercase", "Uppercase",
-                        "Cased", "Case_Ignorable", "Grapheme_Extend"]
-        derived = load_properties("DerivedCoreProperties.txt", want_derived)
-        scripts = load_properties("Scripts.txt", [])
-        props = load_properties("PropList.txt",
-                ["White_Space", "Join_Control", "Noncharacter_Code_Point", "Pattern_White_Space"])
-        norm_props = load_properties("DerivedNormalizationProps.txt",
-                     ["Full_Composition_Exclusion"])
-
-        # category tables
-        for (name, cat, pfuns) in ("general_category", gencats, ["N", "Cc"]), \
-                                  ("derived_property", derived, want_derived), \
-                                  ("property", props, ["White_Space", "Pattern_White_Space"]):
-            emit_property_module(rf, name, cat, pfuns)
-
-        # normalizations and conversions module
-        emit_norm_module(rf, canon_decomp, compat_decomp, combines, norm_props)
-        emit_conversions_module(rf, to_upper, to_lower, to_title)
-    print("Regenerated tables.rs.")
+    main()