From 55efe1fd8eede7b5c0b4e0d323e1620b999952dc Mon Sep 17 00:00:00 2001 From: eryx67 Date: Mon, 23 Dec 2013 15:28:45 +0600 Subject: [PATCH] support unicode --- src/edown_doclet.erl | 14 ++++++++------ src/edown_layout.erl | 20 ++++++++++---------- src/edown_lib.erl | 4 ++-- src/edown_xmerl.erl | 11 ++++++----- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/edown_doclet.erl b/src/edown_doclet.erl index 7dfa5b2..da18370 100644 --- a/src/edown_doclet.erl +++ b/src/edown_doclet.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%============================================================================== %% @author Ulf Wiger -%% @copyright 2010 Erlang Solutions Ltd +%% @copyright 2010 Erlang Solutions Ltd %% @end %% ============================================================================= %% Modified 2012 by Beads Land-Trujillo: get_git_branch/0, redirect_href/3 @@ -52,7 +52,7 @@ %% included in Packages! %% @spec (Command::doclet_gen() | doclet_toc(), edoc_context()) -> ok -%% @doc Main doclet entry point. +%% @doc Main doclet entry point. %% %% Also see {@link //edoc/edoc:layout/2} for layout-related options, and %% {@link //edoc/edoc:get_doc/2} for options related to reading source @@ -139,7 +139,8 @@ gen(Sources, App, Packages, Modules, FileMap, Ctxt) -> ++ lists:concat([modules_frame(Modules1) || Modules1 =/= []]), Text = xmerl:export_simple_content(Data, edown_xmerl), - edoc_lib:write_file(Text, Dir, right_suffix(?INDEX_FILE, Options)), + edoc_lib:write_file(Text, Dir, right_suffix(?INDEX_FILE, Options), '', + [{encoding, utf8}]), edoc_lib:write_info_file(App, Packages, Modules1, Dir), copy_stylesheet(Dir, Options), copy_image(Dir, Options), @@ -210,12 +211,12 @@ redirect_href(Attrs, Branch, BaseHRef) -> {match, _} -> false; nomatch -> - case Href of + case Href of [$# | _] -> HRef1 = do_redirect(?INDEX_FILE ++ Href, AppBlob); _Else -> HRef1 = do_redirect(Href, AppBlob) - end, + end, {true, lists:keyreplace( href, #xmlAttribute.name, Attrs, @@ -294,7 +295,8 @@ source({M, P, Name, Path}, Dir, Suffix, Env, Set, Private, Hidden, true -> Text = edoc:layout(Doc, Options), Name1 = packages_last(M) ++ Suffix, - edoc_lib:write_file(Text, Dir, Name1, P), + edoc_lib:write_file(Text, Dir, Name1, P, + [{encoding, edoc_lib:read_encoding(Name, [])}]), {sets:add_element(Module, Set), Error}; false -> {Set, Error} diff --git a/src/edown_layout.erl b/src/edown_layout.erl index fb43a8e..2d1fbcc 100644 --- a/src/edown_layout.erl +++ b/src/edown_layout.erl @@ -123,7 +123,7 @@ init_opts(Element, Options) -> "" -> R; % don't use any stylesheet S when is_list(S) -> - R#opts{stylesheet = S}; + R#opts{stylesheet = S}; _ -> report("bad value for option `stylesheet'.", []), exit(error) @@ -222,8 +222,8 @@ layout_module(#xmlElement{name = module, content = Es}=E, Opts) -> Res = to_simple(markdown(Title, stylesheet(Opts), Body)), Res. -%% This function is a workaround for a bug in xmerl_lib:expand_content/1 that -%% causes it to lose track of the parents if #xmlElement{} records are +%% This function is a workaround for a bug in xmerl_lib:expand_content/1 that +%% causes it to lose track of the parents if #xmlElement{} records are %% encountered in the structure. %% to_simple([#xmlElement{name = Name, attributes = Attrs, content = Content}|T]) -> @@ -263,7 +263,7 @@ to_simple_attrs(As) -> [{K,V} || #xmlAttribute{name = K, value = V} <- As]. normalize_text(Text) -> - try normalize(binary_to_list(list_to_binary(Text))) + try normalize(Text) catch error:_ -> lists:flatten(io_lib:fwrite("~p", [Text])) @@ -281,7 +281,7 @@ normalize1([]) -> []. to_string(S) -> - binary_to_list(iolist_to_binary([S])). + unicode:characters_to_list([S]). module_params(Es) -> As = [{get_text(argName, Es1), @@ -298,7 +298,7 @@ module_params(Es) -> %% [edoc_lib:datestr(date()), %% edoc_lib:timestr(time())]) %% ]}]}]. - + stylesheet(Opts) -> case Opts#opts.stylesheet of undefined -> @@ -479,7 +479,7 @@ label_anchor(Content, E) -> %% This is currently only done for functions without type spec. -signature(Es, Name) -> +signature(Es, Name) -> [{tt, [Name, "("] ++ seq(fun arg/1, Es) ++ [") -> any()"]}]. arg(#xmlElement{content = Es}) -> @@ -605,7 +605,7 @@ pp_clause(Pre, Type) -> L1 = erl_pp:attribute({attribute,0,spec,{{list_to_atom(Atom),0},[Types]}}), "-spec " ++ L2 = lists:flatten(L1), L3 = Pre ++ lists:nthtail(length(Atom), L2), - re:replace(L3, "\n ", "\n", [{return,list},global]). + re:replace(L3, "\n ", "\n", [{return,list},global,unicode]). format_type(Prefix, Name, Type, Last, #opts{pretty_printer = erl_pp}=Opts) -> try @@ -628,7 +628,7 @@ pp_type(Prefix, Type) -> "::\n" ++ L3 -> {"\n"++L3,6} end, Ss = lists:duplicate(N, $\s), - re:replace(L2, "\n"++Ss, "\n", [{return,list},global]). + re:replace(L2, "\n"++Ss, "\n", [{return,list},global,unicode]). etypef(L, O0) -> {R, O} = etypef(L, [], O0, []), @@ -1253,7 +1253,7 @@ get_first_sentence(Es) -> get_first_sentence_1(Es) -> get_first_sentence_1(Es, []). - + get_first_sentence_1([E = #xmlText{value = Txt} | Es], Acc) -> Last = case Es of [#xmlElement{name = p} | _] -> true; diff --git a/src/edown_lib.erl b/src/edown_lib.erl index 81ca972..bcd0bba 100644 --- a/src/edown_lib.erl +++ b/src/edown_lib.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%============================================================================== %% @author Ulf Wiger -%% @copyright 2010 Erlang Solutions Ltd +%% @copyright 2010 Erlang Solutions Ltd %% @end %% ===================================================================== @@ -65,7 +65,7 @@ redirect_uri(Href, _Name, E) -> case lists:member("/", Href) of false -> [_|_] = URI = get_attrval(href, E), - NewURI = re:replace(URI,".html",".md",[{return,list}]), + NewURI = re:replace(URI,".html",".md",[{return,list},unicode]), replace_uri(NewURI, E); true -> false diff --git a/src/edown_xmerl.erl b/src/edown_xmerl.erl index 2a8abe2..d3a5472 100644 --- a/src/edown_xmerl.erl +++ b/src/edown_xmerl.erl @@ -46,14 +46,15 @@ brstrip(to_string(Text)). to_string(S) -> - binary_to_list(iolist_to_binary([S])). + %%binary_to_list(iolist_to_binary([S])). + unicode:characters_to_list([S]). strip(Str) -> lstrip(rstrip(Str)). -lstrip(Str) -> re:replace(Str,"^\\s","",[]). -rstrip(Str) -> re:replace(Str, "\\s\$", []). +lstrip(Str) -> re:replace(Str,"^\\s","", [unicode]). +rstrip(Str) -> re:replace(Str, "\\s\$", "", [unicode]). % Strip double spaces at end of line -- markdown reads as hard return. -brstrip(Str) -> re:replace(Str, "\\s+\\s\$", "", [global, multiline]). +brstrip(Str) -> re:replace(Str, "\\s+\\s\$", "", [global, multiline, unicode]). %% The '#root#' tag is called when the entire structure has been %% exported. It does not appear in the structure itself. @@ -109,7 +110,7 @@ elem(Tag, Data, Attrs, Parents, E) -> end. escape_pre(Data) -> - re:replace(re:replace(Data, "<", "\\<", [global]), ">", "\\>", [global]). + re:replace(re:replace(Data, "<", "\\<", [global, unicode]), ">", "\\>", [global, unicode]). %% Given content of a pre tag in `Data', entity escape angle brackets %% but leave anchor tags alone. This is less than pretty, but is