Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-21 19:41:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

unicode patches for 5.17

Browse Source 
This includes patches from Christoph Hellwig to split the large data
 tables of the unicode subsystem into a loadable module, which allow
 users to not have them around if case-insensitive filesystems are not to
 be used.  It also includes minor code fixes to unicode and its users,
 from the same author.
 
 There is a trivial conflict in the function encoding_show in
 fs/f2fs/sysfs.c reported by linux-next between commit
 
 84eab2a899 ("f2fs: replace snprintf in show functions with sysfs_emit")
 
 and commit a440943e68 ("unicode: remove the charset field from struct
 unicode_map").  from my tree.
 
 All the patches here have been on linux-next releases for the past
 months.
 
 Signed-off-by: Gabriel Krisman Bertazi <krisman@collabora.com>
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEE8jAUPq50yNjPBCi4QEuZqsMcppQFAmHeLp0ACgkQQEuZqsMc
 ppRWdhAAstuibIlhUj1Vae070P92oaxM/Azz3IgyVFWensJyQV1PvbtFQDhyKM4w
 M3tQ45eK49vVHn+JpLHbiAdZV66rD/sMSsruCVIf/8KNVDisOBQtFar5yxVr0Ion
 AOMoG6/Xrk8BZlZH62fhtJGtu/EFmeFoGVdC81NdTSroe9G+26we3IULwHSE1lNH
 XMJFCgU6otuLDOna16U7kL77Tu7GXRJcQe1+2nRJ+u6Agxy2xTo/s4FHuxzRK0/e
 GsgO1scY6unWM23O6z+qJYazng2Zt3EOZtSGqU4TsvZwjUi2UtAYW1/vAQGc/q3Y
 hGxPYGgKC1VrXLfIcuyng7j0vFPtADbdHMbsJPoyy+Nz4znDJ81IAKAHMO1in3C8
 CHKjW+6InmXNye/uwdRt8Tx49jxUHmWUbQRT5FwMDpzC7MAL+DVdPpVVQgpLVM/H
 gW3YpBEk5qQvVdh8DWZVW3rT3SnMX/v0+u+76FsMHKYNJMNrCnP6vXpCPQl/Gyut
 ycgK7qVF3o/bgNBf072H3ZBZajTv7ePvacP4Wth7m9I2ykk+p4IjQLpTC5rJK0By
 VC1xS4im2VqiIWE9eE5y9cXU1oa/AfOcOF+7FZcxT13IL6hKTtd4+H4yKgdcNsyk
 7RjpGgjp+SU51/EilhEqMFgEe07CURxwGwhApizBSiTIOgZS96U=
 =4q9x
 -----END PGP SIGNATURE-----

Merge tag 'unicode-for-next-5.17' of git://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode

Pull unicode updates from Gabriel Krisman Bertazi:
 "This includes patches from Christoph Hellwig to split the large data
  tables of the unicode subsystem into a loadable module, which allow
  users to not have them around if case-insensitive filesystems are not
  to be used. It also includes minor code fixes to unicode and its
  users, from the same author.

  All the patches here have been on linux-next releases for the past
  months"

* tag 'unicode-for-next-5.17' of git://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode:
  unicode: only export internal symbols for the selftests
  unicode: Add utf8-data module
  unicode: cache the normalization tables in struct unicode_map
  unicode: move utf8cursor to utf8-selftest.c
  unicode: simplify utf8len
  unicode: remove the unused utf8{,n}age{min,max} functions
  unicode: pass a UNICODE_AGE() tripple to utf8_load
  unicode: mark the version field in struct unicode_map unsigned
  unicode: remove the charset field from struct unicode_map
  f2fs: simplify f2fs_sb_read_encoding
  ext4: simplify ext4_sb_read_encoding
This commit is contained in:
Linus Torvalds 2022-01-17 05:40:02 +02:00
commit 6661224e66
12 changed files with 289 additions and 454 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
fs/ext4/super.c
View File

@ -1966,29 +1966,22 @@ static const struct mount_opts {
static const struct ext4_sb_encodings {
__u16 magic;
char *name;
char *version;
unsigned int version;
} ext4_sb_encoding_map[] = {
{EXT4_ENC_UTF8_12_1, "utf8", "12.1.0"},
{EXT4_ENC_UTF8_12_1, "utf8", UNICODE_AGE(12, 1, 0)},
};
static int ext4_sb_read_encoding(const struct ext4_super_block *es,
const struct ext4_sb_encodings **encoding,
__u16 *flags)
static const struct ext4_sb_encodings *
ext4_sb_read_encoding(const struct ext4_super_block *es)
{
__u16 magic = le16_to_cpu(es->s_encoding);
int i;
for (i = 0; i < ARRAY_SIZE(ext4_sb_encoding_map); i++)
if (magic == ext4_sb_encoding_map[i].magic)
break;
return &ext4_sb_encoding_map[i];
if (i >= ARRAY_SIZE(ext4_sb_encoding_map))
return -EINVAL;
*encoding = &ext4_sb_encoding_map[i];
*flags = le16_to_cpu(es->s_encoding_flags);
return 0;
return NULL;
}
#endif
@ -4624,10 +4617,10 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb)
if (ext4_has_feature_casefold(sb) && !sb->s_encoding) {
const struct ext4_sb_encodings *encoding_info;
struct unicode_map *encoding;
__u16 encoding_flags;
__u16 encoding_flags = le16_to_cpu(es->s_encoding_flags);
if (ext4_sb_read_encoding(es, &encoding_info,
&encoding_flags)) {
encoding_info = ext4_sb_read_encoding(es);
if (!encoding_info) {
ext4_msg(sb, KERN_ERR,
"Encoding requested by superblock is unknown");
goto failed_mount;
@ -4636,15 +4629,21 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb)
encoding = utf8_load(encoding_info->version);
if (IS_ERR(encoding)) {
ext4_msg(sb, KERN_ERR,
"can't mount with superblock charset: %s-%s "
"can't mount with superblock charset: %s-%u.%u.%u "
"not supported by the kernel. flags: 0x%x.",
encoding_info->name, encoding_info->version,
encoding_info->name,
unicode_major(encoding_info->version),
unicode_minor(encoding_info->version),
unicode_rev(encoding_info->version),
encoding_flags);
goto failed_mount;
}
ext4_msg(sb, KERN_INFO,"Using encoding defined by superblock: "
"%s-%s with flags 0x%hx", encoding_info->name,
encoding_info->version?:"\b", encoding_flags);
"%s-%u.%u.%u with flags 0x%hx", encoding_info->name,
unicode_major(encoding_info->version),
unicode_minor(encoding_info->version),
unicode_rev(encoding_info->version),
encoding_flags);
sb->s_encoding = encoding;
sb->s_encoding_flags = encoding_flags;

38
fs/f2fs/super.c
View File

@ -260,29 +260,22 @@ void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...)
static const struct f2fs_sb_encodings {
__u16 magic;
char *name;
char *version;
unsigned int version;
} f2fs_sb_encoding_map[] = {
{F2FS_ENC_UTF8_12_1, "utf8", "12.1.0"},
{F2FS_ENC_UTF8_12_1, "utf8", UNICODE_AGE(12, 1, 0)},
};
static int f2fs_sb_read_encoding(const struct f2fs_super_block *sb,
const struct f2fs_sb_encodings **encoding,
__u16 *flags)
static const struct f2fs_sb_encodings *
f2fs_sb_read_encoding(const struct f2fs_super_block *sb)
{
__u16 magic = le16_to_cpu(sb->s_encoding);
int i;
for (i = 0; i < ARRAY_SIZE(f2fs_sb_encoding_map); i++)
if (magic == f2fs_sb_encoding_map[i].magic)
break;
return &f2fs_sb_encoding_map[i];
if (i >= ARRAY_SIZE(f2fs_sb_encoding_map))
return -EINVAL;
*encoding = &f2fs_sb_encoding_map[i];
*flags = le16_to_cpu(sb->s_encoding_flags);
return 0;
return NULL;
}
struct kmem_cache *f2fs_cf_name_slab;
@ -3874,25 +3867,32 @@ static int f2fs_setup_casefold(struct f2fs_sb_info *sbi)
struct unicode_map *encoding;
__u16 encoding_flags;
if (f2fs_sb_read_encoding(sbi->raw_super, &encoding_info,
&encoding_flags)) {
encoding_info = f2fs_sb_read_encoding(sbi->raw_super);
if (!encoding_info) {
f2fs_err(sbi,
"Encoding requested by superblock is unknown");
return -EINVAL;
}
encoding_flags = le16_to_cpu(sbi->raw_super->s_encoding_flags);
encoding = utf8_load(encoding_info->version);
if (IS_ERR(encoding)) {
f2fs_err(sbi,
"can't mount with superblock charset: %s-%s "
"can't mount with superblock charset: %s-%u.%u.%u "
"not supported by the kernel. flags: 0x%x.",
encoding_info->name, encoding_info->version,
encoding_info->name,
unicode_major(encoding_info->version),
unicode_minor(encoding_info->version),
unicode_rev(encoding_info->version),
encoding_flags);
return PTR_ERR(encoding);
}
f2fs_info(sbi, "Using encoding defined by superblock: "
"%s-%s with flags 0x%hx", encoding_info->name,
encoding_info->version?:"\b", encoding_flags);
"%s-%u.%u.%u with flags 0x%hx", encoding_info->name,
unicode_major(encoding_info->version),
unicode_minor(encoding_info->version),
unicode_rev(encoding_info->version),
encoding_flags);
sbi->sb->s_encoding = encoding;
sbi->sb->s_encoding_flags = encoding_flags;

3
fs/f2fs/sysfs.c
View File

@ -196,8 +196,7 @@ static ssize_t encoding_show(struct f2fs_attr *a,
struct super_block *sb = sbi->sb;
if (f2fs_sb_has_casefold(sbi))
return sysfs_emit(buf, "%s (%d.%d.%d)\n",
sb->s_encoding->charset,
return sysfs_emit(buf, "UTF-8 (%d.%d.%d)\n",
(sb->s_encoding->version >> 16) & 0xff,
(sb->s_encoding->version >> 8) & 0xff,
sb->s_encoding->version & 0xff);

13
fs/unicode/Kconfig
View File

@ -8,7 +8,16 @@ config UNICODE
Say Y here to enable UTF-8 NFD normalization and NFD+CF casefolding
support.
config UNICODE_UTF8_DATA
tristate "UTF-8 normalization and casefolding tables"
depends on UNICODE
default UNICODE
help
This contains a large table of case foldings, which can be loaded as
a separate module if you say M here. To be on the safe side stick
to the default of Y. Saying N here makes no sense, if you do not want
utf8 casefolding support, disable CONFIG_UNICODE instead.
config UNICODE_NORMALIZATION_SELFTEST
tristate "Test UTF-8 normalization support"
depends on UNICODE
default n
depends on UNICODE_UTF8_DATA

13
fs/unicode/Makefile
View File

@ -2,14 +2,15 @@
obj-$(CONFIG_UNICODE) += unicode.o
obj-$(CONFIG_UNICODE_NORMALIZATION_SELFTEST) += utf8-selftest.o
obj-$(CONFIG_UNICODE_UTF8_DATA) += utf8data.o
unicode-y := utf8-norm.o utf8-core.o
$(obj)/utf8-norm.o: $(obj)/utf8data.h
$(obj)/utf8-data.o: $(obj)/utf8data.c
# In the normal build, the checked-in utf8data.h is just shipped.
# In the normal build, the checked-in utf8data.c is just shipped.
#
# To generate utf8data.h from UCD, put *.txt files in this directory
# To generate utf8data.c from UCD, put *.txt files in this directory
# and pass REGENERATE_UTF8DATA=1 from the command line.
ifdef REGENERATE_UTF8DATA
@ -24,15 +25,15 @@ quiet_cmd_utf8data = GEN $@
-t $(srctree)/$(src)/NormalizationTest.txt \
-o $@
$(obj)/utf8data.h: $(obj)/mkutf8data $(filter %.txt, $(cmd_utf8data)) FORCE
$(obj)/utf8data.c: $(obj)/mkutf8data $(filter %.txt, $(cmd_utf8data)) FORCE
$(call if_changed,utf8data)
else
$(obj)/utf8data.h: $(src)/utf8data.h_shipped FORCE
$(obj)/utf8data.c: $(src)/utf8data.c_shipped FORCE
$(call if_changed,shipped)
endif
targets += utf8data.h
targets += utf8data.c
hostprogs += mkutf8data

24
fs/unicode/mkutf8data.c
View File

@ -3287,12 +3287,10 @@ static void write_file(void)
open_fail(utf8_name, errno);
fprintf(file, "/* This file is generated code, do not edit. */\n");
fprintf(file, "#ifndef __INCLUDED_FROM_UTF8NORM_C__\n");
fprintf(file, "#error Only nls_utf8-norm.c should include this file.\n");
fprintf(file, "#endif\n");
fprintf(file, "\n");
fprintf(file, "static const unsigned int utf8vers = %#x;\n",
unicode_maxage);
fprintf(file, "#include <linux/module.h>\n");
fprintf(file, "#include <linux/kernel.h>\n");
fprintf(file, "#include \"utf8n.h\"\n");
fprintf(file, "\n");
fprintf(file, "static const unsigned int utf8agetab[] = {\n");
for (i = 0; i != ages_count; i++)
@ -3339,6 +3337,22 @@ static void write_file(void)
fprintf(file, "\n");
}
fprintf(file, "};\n");
fprintf(file, "\n");
fprintf(file, "struct utf8data_table utf8_data_table = {\n");
fprintf(file, "\t.utf8agetab = utf8agetab,\n");
fprintf(file, "\t.utf8agetab_size = ARRAY_SIZE(utf8agetab),\n");
fprintf(file, "\n");
fprintf(file, "\t.utf8nfdicfdata = utf8nfdicfdata,\n");
fprintf(file, "\t.utf8nfdicfdata_size = ARRAY_SIZE(utf8nfdicfdata),\n");
fprintf(file, "\n");
fprintf(file, "\t.utf8nfdidata = utf8nfdidata,\n");
fprintf(file, "\t.utf8nfdidata_size = ARRAY_SIZE(utf8nfdidata),\n");
fprintf(file, "\n");
fprintf(file, "\t.utf8data = utf8data,\n");
fprintf(file, "};\n");
fprintf(file, "EXPORT_SYMBOL_GPL(utf8_data_table);");
fprintf(file, "\n");
fprintf(file, "MODULE_LICENSE(\"GPL v2\");\n");
fclose(file);
}

105
fs/unicode/utf8-core.c
View File

@ -5,16 +5,13 @@
#include <linux/slab.h>
#include <linux/parser.h>
#include <linux/errno.h>
#include <linux/unicode.h>
#include <linux/stringhash.h>
#include "utf8n.h"
int utf8_validate(const struct unicode_map *um, const struct qstr *str)
{
const struct utf8data *data = utf8nfdi(um->version);
if (utf8nlen(data, str->name, str->len) < 0)
if (utf8nlen(um, UTF8_NFDI, str->name, str->len) < 0)
return -1;
return 0;
}
@ -23,14 +20,13 @@ EXPORT_SYMBOL(utf8_validate);
int utf8_strncmp(const struct unicode_map *um,
const struct qstr *s1, const struct qstr *s2)
{
const struct utf8data *data = utf8nfdi(um->version);
struct utf8cursor cur1, cur2;
int c1, c2;
if (utf8ncursor(&cur1, data, s1->name, s1->len) < 0)
if (utf8ncursor(&cur1, um, UTF8_NFDI, s1->name, s1->len) < 0)
return -EINVAL;
if (utf8ncursor(&cur2, data, s2->name, s2->len) < 0)
if (utf8ncursor(&cur2, um, UTF8_NFDI, s2->name, s2->len) < 0)
return -EINVAL;
do {
@ -50,14 +46,13 @@ EXPORT_SYMBOL(utf8_strncmp);
int utf8_strncasecmp(const struct unicode_map *um,
const struct qstr *s1, const struct qstr *s2)
{
const struct utf8data *data = utf8nfdicf(um->version);
struct utf8cursor cur1, cur2;
int c1, c2;
if (utf8ncursor(&cur1, data, s1->name, s1->len) < 0)
if (utf8ncursor(&cur1, um, UTF8_NFDICF, s1->name, s1->len) < 0)
return -EINVAL;
if (utf8ncursor(&cur2, data, s2->name, s2->len) < 0)
if (utf8ncursor(&cur2, um, UTF8_NFDICF, s2->name, s2->len) < 0)
return -EINVAL;
do {
@ -81,12 +76,11 @@ int utf8_strncasecmp_folded(const struct unicode_map *um,
const struct qstr *cf,
const struct qstr *s1)
{
const struct utf8data *data = utf8nfdicf(um->version);
struct utf8cursor cur1;
int c1, c2;
int i = 0;
if (utf8ncursor(&cur1, data, s1->name, s1->len) < 0)
if (utf8ncursor(&cur1, um, UTF8_NFDICF, s1->name, s1->len) < 0)
return -EINVAL;
do {
@ -105,11 +99,10 @@ EXPORT_SYMBOL(utf8_strncasecmp_folded);
int utf8_casefold(const struct unicode_map *um, const struct qstr *str,
unsigned char *dest, size_t dlen)
{
const struct utf8data *data = utf8nfdicf(um->version);
struct utf8cursor cur;
size_t nlen = 0;
if (utf8ncursor(&cur, data, str->name, str->len) < 0)
if (utf8ncursor(&cur, um, UTF8_NFDICF, str->name, str->len) < 0)
return -EINVAL;
for (nlen = 0; nlen < dlen; nlen++) {
@ -128,12 +121,11 @@ EXPORT_SYMBOL(utf8_casefold);
int utf8_casefold_hash(const struct unicode_map *um, const void *salt,
struct qstr *str)
{
const struct utf8data *data = utf8nfdicf(um->version);
struct utf8cursor cur;
int c;
unsigned long hash = init_name_hash(salt);
if (utf8ncursor(&cur, data, str->name, str->len) < 0)
if (utf8ncursor(&cur, um, UTF8_NFDICF, str->name, str->len) < 0)
return -EINVAL;
while ((c = utf8byte(&cur))) {
@ -149,11 +141,10 @@ EXPORT_SYMBOL(utf8_casefold_hash);
int utf8_normalize(const struct unicode_map *um, const struct qstr *str,
unsigned char *dest, size_t dlen)
{
const struct utf8data *data = utf8nfdi(um->version);
struct utf8cursor cur;
ssize_t nlen = 0;
if (utf8ncursor(&cur, data, str->name, str->len) < 0)
if (utf8ncursor(&cur, um, UTF8_NFDI, str->name, str->len) < 0)
return -EINVAL;
for (nlen = 0; nlen < dlen; nlen++) {
@ -167,69 +158,59 @@ int utf8_normalize(const struct unicode_map *um, const struct qstr *str,
}
return -EINVAL;
}
EXPORT_SYMBOL(utf8_normalize);
static int utf8_parse_version(const char *version, unsigned int *maj,
unsigned int *min, unsigned int *rev)
static const struct utf8data *find_table_version(const struct utf8data *table,
size_t nr_entries, unsigned int version)
{
substring_t args[3];
char version_string[12];
static const struct match_token token[] = {
{1, "%d.%d.%d"},
{0, NULL}
};
size_t i = nr_entries - 1;
strncpy(version_string, version, sizeof(version_string));
if (match_token(version_string, token, args) != 1)
return -EINVAL;
if (match_int(&args[0], maj) || match_int(&args[1], min) ||
match_int(&args[2], rev))
return -EINVAL;
return 0;
while (version < table[i].maxage)
i--;
if (version > table[i].maxage)
return NULL;
return &table[i];
}
struct unicode_map *utf8_load(const char *version)
struct unicode_map *utf8_load(unsigned int version)
{
struct unicode_map *um = NULL;
int unicode_version;
if (version) {
unsigned int maj, min, rev;
if (utf8_parse_version(version, &maj, &min, &rev) < 0)
return ERR_PTR(-EINVAL);
if (!utf8version_is_supported(maj, min, rev))
return ERR_PTR(-EINVAL);
unicode_version = UNICODE_AGE(maj, min, rev);
} else {
unicode_version = utf8version_latest();
printk(KERN_WARNING"UTF-8 version not specified. "
"Assuming latest supported version (%d.%d.%d).",
(unicode_version >> 16) & 0xff,
(unicode_version >> 8) & 0xff,
(unicode_version & 0xff));
}
struct unicode_map *um;
um = kzalloc(sizeof(struct unicode_map), GFP_KERNEL);
if (!um)
return ERR_PTR(-ENOMEM);
um->version = version;
um->charset = "UTF-8";
um->version = unicode_version;
um->tables = symbol_request(utf8_data_table);
if (!um->tables)
goto out_free_um;
if (!utf8version_is_supported(um, version))
goto out_symbol_put;
um->ntab[UTF8_NFDI] = find_table_version(um->tables->utf8nfdidata,
um->tables->utf8nfdidata_size, um->version);
if (!um->ntab[UTF8_NFDI])
goto out_symbol_put;
um->ntab[UTF8_NFDICF] = find_table_version(um->tables->utf8nfdicfdata,
um->tables->utf8nfdicfdata_size, um->version);
if (!um->ntab[UTF8_NFDICF])
goto out_symbol_put;
return um;
out_symbol_put:
symbol_put(um->tables);
out_free_um:
kfree(um);
return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL(utf8_load);
void utf8_unload(struct unicode_map *um)
{
kfree(um);
if (um) {
symbol_put(utf8_data_table);
kfree(um);
}
}
EXPORT_SYMBOL(utf8_unload);

262
fs/unicode/utf8-norm.c
View File

@ -6,34 +6,17 @@
#include "utf8n.h"
struct utf8data {
unsigned int maxage;
unsigned int offset;
};
#define __INCLUDED_FROM_UTF8NORM_C__
#include "utf8data.h"
#undef __INCLUDED_FROM_UTF8NORM_C__
int utf8version_is_supported(u8 maj, u8 min, u8 rev)
int utf8version_is_supported(const struct unicode_map *um, unsigned int version)
{
int i = ARRAY_SIZE(utf8agetab) - 1;
unsigned int sb_utf8version = UNICODE_AGE(maj, min, rev);
int i = um->tables->utf8agetab_size - 1;
while (i >= 0 && utf8agetab[i] != 0) {
if (sb_utf8version == utf8agetab[i])
while (i >= 0 && um->tables->utf8agetab[i] != 0) {
if (version == um->tables->utf8agetab[i])
return 1;
i--;
}
return 0;
}
EXPORT_SYMBOL(utf8version_is_supported);
int utf8version_latest(void)
{
return utf8vers;
}
EXPORT_SYMBOL(utf8version_latest);
/*
* UTF-8 valid ranges.
@ -168,7 +151,7 @@ typedef const unsigned char utf8trie_t;
* underlying datatype: unsigned char.
*
* leaf[0]: The unicode version, stored as a generation number that is
* an index into utf8agetab[]. With this we can filter code
* an index into ->utf8agetab[]. With this we can filter code
* points based on the unicode version in which they were
* defined. The CCC of a non-defined code point is 0.
* leaf[1]: Canonical Combining Class. During normalization, we need
@ -316,21 +299,19 @@ utf8hangul(const char *str, unsigned char *hangul)
* is well-formed and corresponds to a known unicode code point. The
* shorthand for this will be "is valid UTF-8 unicode".
*/
static utf8leaf_t *utf8nlookup(const struct utf8data *data,
unsigned char *hangul, const char *s, size_t len)
static utf8leaf_t *utf8nlookup(const struct unicode_map *um,
enum utf8_normalization n, unsigned char *hangul, const char *s,
size_t len)
{
utf8trie_t *trie = NULL;
utf8trie_t *trie = um->tables->utf8data + um->ntab[n]->offset;
int offlen;
int offset;
int mask;
int node;
if (!data)
return NULL;
if (len == 0)
return NULL;
trie = utf8data + data->offset;
node = 1;
while (node) {
offlen = (*trie & OFFLEN) >> OFFLEN_SHIFT;
@ -392,172 +373,29 @@ static utf8leaf_t *utf8nlookup(const struct utf8data *data,
*
* Forwards to utf8nlookup().
*/
static utf8leaf_t *utf8lookup(const struct utf8data *data,
unsigned char *hangul, const char *s)
static utf8leaf_t *utf8lookup(const struct unicode_map *um,
enum utf8_normalization n, unsigned char *hangul, const char *s)
{
return utf8nlookup(data, hangul, s, (size_t)-1);
return utf8nlookup(um, n, hangul, s, (size_t)-1);
}
/*
* Maximum age of any character in s.
* Return -1 if s is not valid UTF-8 unicode.
* Return 0 if only non-assigned code points are used.
*/
int utf8agemax(const struct utf8data *data, const char *s)
{
utf8leaf_t *leaf;
int age = 0;
int leaf_age;
unsigned char hangul[UTF8HANGULLEAF];
if (!data)
return -1;
while (*s) {
leaf = utf8lookup(data, hangul, s);
if (!leaf)
return -1;
leaf_age = utf8agetab[LEAF_GEN(leaf)];
if (leaf_age <= data->maxage && leaf_age > age)
age = leaf_age;
s += utf8clen(s);
}
return age;
}
EXPORT_SYMBOL(utf8agemax);
/*
* Minimum age of any character in s.
* Return -1 if s is not valid UTF-8 unicode.
* Return 0 if non-assigned code points are used.
*/
int utf8agemin(const struct utf8data *data, const char *s)
{
utf8leaf_t *leaf;
int age;
int leaf_age;
unsigned char hangul[UTF8HANGULLEAF];
if (!data)
return -1;
age = data->maxage;
while (*s) {
leaf = utf8lookup(data, hangul, s);
if (!leaf)
return -1;
leaf_age = utf8agetab[LEAF_GEN(leaf)];
if (leaf_age <= data->maxage && leaf_age < age)
age = leaf_age;
s += utf8clen(s);
}
return age;
}
EXPORT_SYMBOL(utf8agemin);
/*
* Maximum age of any character in s, touch at most len bytes.
* Return -1 if s is not valid UTF-8 unicode.
*/
int utf8nagemax(const struct utf8data *data, const char *s, size_t len)
{
utf8leaf_t *leaf;
int age = 0;
int leaf_age;
unsigned char hangul[UTF8HANGULLEAF];
if (!data)
return -1;
while (len && *s) {
leaf = utf8nlookup(data, hangul, s, len);
if (!leaf)
return -1;
leaf_age = utf8agetab[LEAF_GEN(leaf)];
if (leaf_age <= data->maxage && leaf_age > age)
age = leaf_age;
len -= utf8clen(s);
s += utf8clen(s);
}
return age;
}
EXPORT_SYMBOL(utf8nagemax);
/*
* Maximum age of any character in s, touch at most len bytes.
* Return -1 if s is not valid UTF-8 unicode.
*/
int utf8nagemin(const struct utf8data *data, const char *s, size_t len)
{
utf8leaf_t *leaf;
int leaf_age;
int age;
unsigned char hangul[UTF8HANGULLEAF];
if (!data)
return -1;
age = data->maxage;
while (len && *s) {
leaf = utf8nlookup(data, hangul, s, len);
if (!leaf)
return -1;
leaf_age = utf8agetab[LEAF_GEN(leaf)];
if (leaf_age <= data->maxage && leaf_age < age)
age = leaf_age;
len -= utf8clen(s);
s += utf8clen(s);
}
return age;
}
EXPORT_SYMBOL(utf8nagemin);
/*
* Length of the normalization of s.
* Return -1 if s is not valid UTF-8 unicode.
*
* A string of Default_Ignorable_Code_Point has length 0.
*/
ssize_t utf8len(const struct utf8data *data, const char *s)
{
utf8leaf_t *leaf;
size_t ret = 0;
unsigned char hangul[UTF8HANGULLEAF];
if (!data)
return -1;
while (*s) {
leaf = utf8lookup(data, hangul, s);
if (!leaf)
return -1;
if (utf8agetab[LEAF_GEN(leaf)] > data->maxage)
ret += utf8clen(s);
else if (LEAF_CCC(leaf) == DECOMPOSE)
ret += strlen(LEAF_STR(leaf));
else
ret += utf8clen(s);
s += utf8clen(s);
}
return ret;
}
EXPORT_SYMBOL(utf8len);
/*
* Length of the normalization of s, touch at most len bytes.
* Return -1 if s is not valid UTF-8 unicode.
*/
ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len)
ssize_t utf8nlen(const struct unicode_map *um, enum utf8_normalization n,
const char *s, size_t len)
{
utf8leaf_t *leaf;
size_t ret = 0;
unsigned char hangul[UTF8HANGULLEAF];
if (!data)
return -1;
while (len && *s) {
leaf = utf8nlookup(data, hangul, s, len);
leaf = utf8nlookup(um, n, hangul, s, len);
if (!leaf)
return -1;
if (utf8agetab[LEAF_GEN(leaf)] > data->maxage)
if (um->tables->utf8agetab[LEAF_GEN(leaf)] >
um->ntab[n]->maxage)
ret += utf8clen(s);
else if (LEAF_CCC(leaf) == DECOMPOSE)
ret += strlen(LEAF_STR(leaf));
@ -568,7 +406,6 @@ ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len)
}
return ret;
}
EXPORT_SYMBOL(utf8nlen);
/*
* Set up an utf8cursor for use by utf8byte().
@ -580,14 +417,13 @@ EXPORT_SYMBOL(utf8nlen);
*
* Returns -1 on error, 0 on success.
*/
int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data,
const char *s, size_t len)
int utf8ncursor(struct utf8cursor *u8c, const struct unicode_map *um,
enum utf8_normalization n, const char *s, size_t len)
{
if (!data)
return -1;
if (!s)
return -1;
u8c->data = data;
u8c->um = um;
u8c->n = n;
u8c->s = s;
u8c->p = NULL;
u8c->ss = NULL;
@ -604,23 +440,6 @@ int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data,
return -1;
return 0;
}
EXPORT_SYMBOL(utf8ncursor);
/*
* Set up an utf8cursor for use by utf8byte().
*
* u8c : pointer to cursor.
* data : const struct utf8data to use for normalization.
* s : NUL-terminated string.
*
* Returns -1 on error, 0 on success.
*/
int utf8cursor(struct utf8cursor *u8c, const struct utf8data *data,
const char *s)
{
return utf8ncursor(u8c, data, s, (unsigned int)-1);
}
EXPORT_SYMBOL(utf8cursor);
/*
* Get one byte from the normalized form of the string described by u8c.
@ -678,9 +497,9 @@ int utf8byte(struct utf8cursor *u8c)
/* Look up the data for the current character. */
if (u8c->p) {
leaf = utf8lookup(u8c->data, u8c->hangul, u8c->s);
leaf = utf8lookup(u8c->um, u8c->n, u8c->hangul, u8c->s);
} else {
leaf = utf8nlookup(u8c->data, u8c->hangul,
leaf = utf8nlookup(u8c->um, u8c->n, u8c->hangul,
u8c->s, u8c->len);
}
@ -690,7 +509,8 @@ int utf8byte(struct utf8cursor *u8c)
ccc = LEAF_CCC(leaf);
/* Characters that are too new have CCC 0. */
if (utf8agetab[LEAF_GEN(leaf)] > u8c->data->maxage) {
if (u8c->um->tables->utf8agetab[LEAF_GEN(leaf)] >
u8c->um->ntab[u8c->n]->maxage) {
ccc = STOPPER;
} else if (ccc == DECOMPOSE) {
u8c->len -= utf8clen(u8c->s);
@ -704,7 +524,7 @@ int utf8byte(struct utf8cursor *u8c)
goto ccc_mismatch;
}
leaf = utf8lookup(u8c->data, u8c->hangul, u8c->s);
leaf = utf8lookup(u8c->um, u8c->n, u8c->hangul, u8c->s);
if (!leaf)
return -1;
ccc = LEAF_CCC(leaf);
@ -765,28 +585,10 @@ ccc_mismatch:
}
}
}
EXPORT_SYMBOL(utf8byte);
const struct utf8data *utf8nfdi(unsigned int maxage)
{
int i = ARRAY_SIZE(utf8nfdidata) - 1;
while (maxage < utf8nfdidata[i].maxage)
i--;
if (maxage > utf8nfdidata[i].maxage)
return NULL;
return &utf8nfdidata[i];
}
EXPORT_SYMBOL(utf8nfdi);
const struct utf8data *utf8nfdicf(unsigned int maxage)
{
int i = ARRAY_SIZE(utf8nfdicfdata) - 1;
while (maxage < utf8nfdicfdata[i].maxage)
i--;
if (maxage > utf8nfdicfdata[i].maxage)
return NULL;
return &utf8nfdicfdata[i];
}
EXPORT_SYMBOL(utf8nfdicf);
#ifdef CONFIG_UNICODE_NORMALIZATION_SELFTEST_MODULE
EXPORT_SYMBOL_GPL(utf8version_is_supported);
EXPORT_SYMBOL_GPL(utf8nlen);
EXPORT_SYMBOL_GPL(utf8ncursor);
EXPORT_SYMBOL_GPL(utf8byte);
#endif

94
fs/unicode/utf8-selftest.c
View File

@ -18,9 +18,7 @@ unsigned int failed_tests;
unsigned int total_tests;
/* Tests will be based on this version. */
#define latest_maj 12
#define latest_min 1
#define latest_rev 0
#define UTF8_LATEST UNICODE_AGE(12, 1, 0)
#define _test(cond, func, line, fmt, ...) do { \
total_tests++; \
@ -160,18 +158,22 @@ static const struct {
}
};
static void check_utf8_nfdi(void)
static ssize_t utf8len(const struct unicode_map *um, enum utf8_normalization n,
const char *s)
{
return utf8nlen(um, n, s, (size_t)-1);
}
static int utf8cursor(struct utf8cursor *u8c, const struct unicode_map *um,
enum utf8_normalization n, const char *s)
{
return utf8ncursor(u8c, um, n, s, (unsigned int)-1);
}
static void check_utf8_nfdi(struct unicode_map *um)
{
int i;
struct utf8cursor u8c;
const struct utf8data *data;
data = utf8nfdi(UNICODE_AGE(latest_maj, latest_min, latest_rev));
if (!data) {
pr_err("%s: Unable to load utf8-%d.%d.%d. Skipping.\n",
__func__, latest_maj, latest_min, latest_rev);
return;
}
for (i = 0; i < ARRAY_SIZE(nfdi_test_data); i++) {
int len = strlen(nfdi_test_data[i].str);
@ -179,10 +181,11 @@ static void check_utf8_nfdi(void)
int j = 0;
unsigned char c;
test((utf8len(data, nfdi_test_data[i].str) == nlen));
test((utf8nlen(data, nfdi_test_data[i].str, len) == nlen));
test((utf8len(um, UTF8_NFDI, nfdi_test_data[i].str) == nlen));
test((utf8nlen(um, UTF8_NFDI, nfdi_test_data[i].str, len) ==
nlen));
if (utf8cursor(&u8c, data, nfdi_test_data[i].str) < 0)
if (utf8cursor(&u8c, um, UTF8_NFDI, nfdi_test_data[i].str) < 0)
pr_err("can't create cursor\n");
while ((c = utf8byte(&u8c)) > 0) {
@ -196,18 +199,10 @@ static void check_utf8_nfdi(void)
}
}
static void check_utf8_nfdicf(void)
static void check_utf8_nfdicf(struct unicode_map *um)
{
int i;
struct utf8cursor u8c;
const struct utf8data *data;
data = utf8nfdicf(UNICODE_AGE(latest_maj, latest_min, latest_rev));
if (!data) {
pr_err("%s: Unable to load utf8-%d.%d.%d. Skipping.\n",
__func__, latest_maj, latest_min, latest_rev);
return;
}
for (i = 0; i < ARRAY_SIZE(nfdicf_test_data); i++) {
int len = strlen(nfdicf_test_data[i].str);
@ -215,10 +210,13 @@ static void check_utf8_nfdicf(void)
int j = 0;
unsigned char c;
test((utf8len(data, nfdicf_test_data[i].str) == nlen));
test((utf8nlen(data, nfdicf_test_data[i].str, len) == nlen));
test((utf8len(um, UTF8_NFDICF, nfdicf_test_data[i].str) ==
nlen));
test((utf8nlen(um, UTF8_NFDICF, nfdicf_test_data[i].str, len) ==
nlen));
if (utf8cursor(&u8c, data, nfdicf_test_data[i].str) < 0)
if (utf8cursor(&u8c, um, UTF8_NFDICF,
nfdicf_test_data[i].str) < 0)
pr_err("can't create cursor\n");
while ((c = utf8byte(&u8c)) > 0) {
@ -232,16 +230,9 @@ static void check_utf8_nfdicf(void)
}
}
static void check_utf8_comparisons(void)
static void check_utf8_comparisons(struct unicode_map *table)
{
int i;
struct unicode_map *table = utf8_load("12.1.0");
if (IS_ERR(table)) {
pr_err("%s: Unable to load utf8 %d.%d.%d. Skipping.\n",
__func__, latest_maj, latest_min, latest_rev);
return;
}
for (i = 0; i < ARRAY_SIZE(nfdi_test_data); i++) {
const struct qstr s1 = {.name = nfdi_test_data[i].str,
@ -262,42 +253,49 @@ static void check_utf8_comparisons(void)
test_f(!utf8_strncasecmp(table, &s1, &s2),
"%s %s comparison mismatch\n", s1.name, s2.name);
}
utf8_unload(table);
}
static void check_supported_versions(void)
static void check_supported_versions(struct unicode_map *um)
{
/* Unicode 7.0.0 should be supported. */
test(utf8version_is_supported(7, 0, 0));
test(utf8version_is_supported(um, UNICODE_AGE(7, 0, 0)));
/* Unicode 9.0.0 should be supported. */
test(utf8version_is_supported(9, 0, 0));
test(utf8version_is_supported(um, UNICODE_AGE(9, 0, 0)));
/* Unicode 1x.0.0 (the latest version) should be supported. */
test(utf8version_is_supported(latest_maj, latest_min, latest_rev));
test(utf8version_is_supported(um, UTF8_LATEST));
/* Next versions don't exist. */
test(!utf8version_is_supported(13, 0, 0));
test(!utf8version_is_supported(0, 0, 0));
test(!utf8version_is_supported(-1, -1, -1));
test(!utf8version_is_supported(um, UNICODE_AGE(13, 0, 0)));
test(!utf8version_is_supported(um, UNICODE_AGE(0, 0, 0)));
test(!utf8version_is_supported(um, UNICODE_AGE(-1, -1, -1)));
}
static int __init init_test_ucd(void)
{
struct unicode_map *um;
failed_tests = 0;
total_tests = 0;
check_supported_versions();
check_utf8_nfdi();
check_utf8_nfdicf();
check_utf8_comparisons();
um = utf8_load(UTF8_LATEST);
if (IS_ERR(um)) {
pr_err("%s: Unable to load utf8 table.\n", __func__);
return PTR_ERR(um);
}
check_supported_versions(um);
check_utf8_nfdi(um);
check_utf8_nfdicf(um);
check_utf8_comparisons(um);
if (!failed_tests)
pr_info("All %u tests passed\n", total_tests);
else
pr_err("%u out of %u tests failed\n", failed_tests,
total_tests);
utf8_unload(um);
return 0;
}

22
fs/unicode/utf8data.h_shipped → fs/unicode/utf8data.c_shipped
View File

@ -1,9 +1,8 @@
/* This file is generated code, do not edit. */
#ifndef __INCLUDED_FROM_UTF8NORM_C__
#error Only nls_utf8-norm.c should include this file.
#endif
static const unsigned int utf8vers = 0xc0100;
#include <linux/module.h>
#include <linux/kernel.h>
#include "utf8n.h"
static const unsigned int utf8agetab[] = {
0,
@ -4107,3 +4106,18 @@ static const unsigned char utf8data[64256] = {
0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf,0x86,0xcf,0x06,0x02,0x00,
0x81,0x80,0xcf,0x86,0x85,0x84,0xcf,0x86,0xcf,0x06,0x02,0x00,0x00,0x00,0x00,0x00
};
struct utf8data_table utf8_data_table = {
.utf8agetab = utf8agetab,
.utf8agetab_size = ARRAY_SIZE(utf8agetab),
.utf8nfdicfdata = utf8nfdicfdata,
.utf8nfdicfdata_size = ARRAY_SIZE(utf8nfdicfdata),
.utf8nfdidata = utf8nfdidata,
.utf8nfdidata_size = ARRAY_SIZE(utf8nfdidata),
.utf8data = utf8data,
};
EXPORT_SYMBOL_GPL(utf8_data_table);
MODULE_LICENSE("GPL v2");

81
fs/unicode/utf8n.h
View File

@ -11,53 +11,9 @@
#include <linux/export.h>
#include <linux/string.h>
#include <linux/module.h>
#include <linux/unicode.h>
/* Encoding a unicode version number as a single unsigned int. */
#define UNICODE_MAJ_SHIFT (16)
#define UNICODE_MIN_SHIFT (8)
#define UNICODE_AGE(MAJ, MIN, REV) \
(((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \
((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \
((unsigned int)(REV)))
/* Highest unicode version supported by the data tables. */
extern int utf8version_is_supported(u8 maj, u8 min, u8 rev);
extern int utf8version_latest(void);
/*
* Look for the correct const struct utf8data for a unicode version.
* Returns NULL if the version requested is too new.
*
* Two normalization forms are supported: nfdi and nfdicf.
*
* nfdi:
* - Apply unicode normalization form NFD.
* - Remove any Default_Ignorable_Code_Point.
*
* nfdicf:
* - Apply unicode normalization form NFD.
* - Remove any Default_Ignorable_Code_Point.
* - Apply a full casefold (C + F).
*/
extern const struct utf8data *utf8nfdi(unsigned int maxage);
extern const struct utf8data *utf8nfdicf(unsigned int maxage);
/*
* Determine the maximum age of any unicode character in the string.
* Returns 0 if only unassigned code points are present.
* Returns -1 if the input is not valid UTF-8.
*/
extern int utf8agemax(const struct utf8data *data, const char *s);
extern int utf8nagemax(const struct utf8data *data, const char *s, size_t len);
/*
* Determine the minimum age of any unicode character in the string.
* Returns 0 if any unassigned code points are present.
* Returns -1 if the input is not valid UTF-8.
*/
extern int utf8agemin(const struct utf8data *data, const char *s);
extern int utf8nagemin(const struct utf8data *data, const char *s, size_t len);
int utf8version_is_supported(const struct unicode_map *um, unsigned int version);
/*
* Determine the length of the normalized from of the string,
@ -65,8 +21,8 @@ extern int utf8nagemin(const struct utf8data *data, const char *s, size_t len);
* Returns 0 if only ignorable code points are present.
* Returns -1 if the input is not valid UTF-8.
*/
extern ssize_t utf8len(const struct utf8data *data, const char *s);
extern ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len);
ssize_t utf8nlen(const struct unicode_map *um, enum utf8_normalization n,
const char *s, size_t len);
/* Needed in struct utf8cursor below. */
#define UTF8HANGULLEAF (12)
@ -75,7 +31,8 @@ extern ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len);
* Cursor structure used by the normalizer.
*/
struct utf8cursor {
const struct utf8data *data;
const struct unicode_map *um;
enum utf8_normalization n;
const char *s;
const char *p;
const char *ss;
@ -92,10 +49,8 @@ struct utf8cursor {
* Returns 0 on success.
* Returns -1 on failure.
*/
extern int utf8cursor(struct utf8cursor *u8c, const struct utf8data *data,
const char *s);
extern int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data,
const char *s, size_t len);
int utf8ncursor(struct utf8cursor *u8c, const struct unicode_map *um,
enum utf8_normalization n, const char *s, size_t len);
/*
* Get the next byte in the normalization.
@ -105,4 +60,24 @@ extern int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data,
*/
extern int utf8byte(struct utf8cursor *u8c);
struct utf8data {
unsigned int maxage;
unsigned int offset;
};
struct utf8data_table {
const unsigned int *utf8agetab;
int utf8agetab_size;
const struct utf8data *utf8nfdicfdata;
int utf8nfdicfdata_size;
const struct utf8data *utf8nfdidata;
int utf8nfdidata_size;
const unsigned char *utf8data;
};
extern struct utf8data_table utf8_data_table;
#endif /* UTF8NORM_H */

49
include/linux/unicode.h
View File

@ -5,9 +5,52 @@
#include <linux/init.h>
#include <linux/dcache.h>
struct utf8data;
struct utf8data_table;
#define UNICODE_MAJ_SHIFT 16
#define UNICODE_MIN_SHIFT 8
#define UNICODE_AGE(MAJ, MIN, REV) \
(((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \
((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \
((unsigned int)(REV)))
static inline u8 unicode_major(unsigned int age)
{
return (age >> UNICODE_MAJ_SHIFT) & 0xff;
}
static inline u8 unicode_minor(unsigned int age)
{
return (age >> UNICODE_MIN_SHIFT) & 0xff;
}
static inline u8 unicode_rev(unsigned int age)
{
return age & 0xff;
}
/*
* Two normalization forms are supported:
* 1) NFDI
* - Apply unicode normalization form NFD.
* - Remove any Default_Ignorable_Code_Point.
* 2) NFDICF
* - Apply unicode normalization form NFD.
* - Remove any Default_Ignorable_Code_Point.
* - Apply a full casefold (C + F).
*/
enum utf8_normalization {
UTF8_NFDI = 0,
UTF8_NFDICF,
UTF8_NMAX,
};
struct unicode_map {
const char *charset;
int version;
unsigned int version;
const struct utf8data *ntab[UTF8_NMAX];
const struct utf8data_table *tables;
};
int utf8_validate(const struct unicode_map *um, const struct qstr *str);
@ -30,7 +73,7 @@ int utf8_casefold(const struct unicode_map *um, const struct qstr *str,
int utf8_casefold_hash(const struct unicode_map *um, const void *salt,
struct qstr *str);
struct unicode_map *utf8_load(const char *version);
struct unicode_map *utf8_load(unsigned int version);
void utf8_unload(struct unicode_map *um);
#endif /* _LINUX_UNICODE_H */