Merge pull request uwiger#27 from eryx67/master

support unicode

src/edown_doclet.erl

@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%==============================================================================
 %% @author Ulf Wiger <[email protected]>
-%% @copyright 2010 Erlang Solutions Ltd 
+%% @copyright 2010 Erlang Solutions Ltd
 %% @end
 %% =============================================================================
 %% Modified 2012 by Beads Land-Trujillo:  get_git_branch/0, redirect_href/3
@@ -52,7 +52,7 @@
 %% included in Packages!
 
 %% @spec (Command::doclet_gen() | doclet_toc(), edoc_context()) -> ok
-%% @doc Main doclet entry point. 
+%% @doc Main doclet entry point.
 %%
 %% Also see {@link //edoc/edoc:layout/2} for layout-related options, and
 %% {@link //edoc/edoc:get_doc/2} for options related to reading source
@@ -139,7 +139,8 @@ gen(Sources, App, Packages, Modules, FileMap, Ctxt) ->
 	 ++ lists:concat([modules_frame(Modules1) || Modules1 =/= []]),
 
     Text = xmerl:export_simple_content(Data, edown_xmerl),
-    edoc_lib:write_file(Text, Dir, right_suffix(?INDEX_FILE, Options)),
+    edoc_lib:write_file(Text, Dir, right_suffix(?INDEX_FILE, Options), '',
+                        [{encoding, utf8}]),
     edoc_lib:write_info_file(App, Packages, Modules1, Dir),
     copy_stylesheet(Dir, Options),
     copy_image(Dir, Options),
@@ -210,12 +211,12 @@ redirect_href(Attrs, Branch, BaseHRef) ->
 		{match, _} ->
 		    false;
 		nomatch ->
-			case Href of 
+			case Href of
 				[$# | _]	->
 					HRef1 = do_redirect(?INDEX_FILE ++ Href, AppBlob);
 				_Else ->
 					HRef1 = do_redirect(Href, AppBlob)
-			end,			
+			end,
 		    {true,
 		     lists:keyreplace(
 		       href, #xmlAttribute.name, Attrs,
@@ -294,7 +295,8 @@ source({M, P, Name, Path}, Dir, Suffix, Env, Set, Private, Hidden,
 		true ->
 		    Text = edoc:layout(Doc, Options),
 		    Name1 = packages_last(M) ++ Suffix,
-		    edoc_lib:write_file(Text, Dir, Name1, P),
+		    edoc_lib:write_file(Text, Dir, Name1, P,
+                               [{encoding, edoc_lib:read_encoding(Name, [])}]),
 		    {sets:add_element(Module, Set), Error};
 		false ->
 		    {Set, Error}

src/edown_layout.erl

@@ -123,7 +123,7 @@ init_opts(Element, Options) ->
 	"" ->
 	    R;  % don't use any stylesheet
 	S when is_list(S) ->
-	    R#opts{stylesheet = S}; 
+	    R#opts{stylesheet = S};
 	_ ->
 	    report("bad value for option `stylesheet'.", []),
 	    exit(error)
@@ -222,8 +222,8 @@ layout_module(#xmlElement{name = module, content = Es}=E, Opts) ->
     Res = to_simple(markdown(Title, stylesheet(Opts), Body)),
     Res.
 
-%% This function is a workaround for a bug in xmerl_lib:expand_content/1 that 
-%% causes it to lose track of the parents if #xmlElement{} records are 
+%% This function is a workaround for a bug in xmerl_lib:expand_content/1 that
+%% causes it to lose track of the parents if #xmlElement{} records are
 %% encountered in the structure.
 %%
 to_simple([#xmlElement{name = Name, attributes = Attrs, content = Content}|T]) ->
@@ -263,7 +263,7 @@ to_simple_attrs(As) ->
     [{K,V} || #xmlAttribute{name = K, value = V} <- As].
 
 normalize_text(Text) ->
-    try normalize(binary_to_list(list_to_binary(Text)))
+    try normalize(Text)
     catch
 	error:_ ->
 	    lists:flatten(io_lib:fwrite("~p", [Text]))
@@ -281,7 +281,7 @@ normalize1([]) ->
     [].
 
 to_string(S) ->
-    binary_to_list(iolist_to_binary([S])).
+    unicode:characters_to_list([S]).
 
 module_params(Es) ->
     As = [{get_text(argName, Es1),
@@ -298,7 +298,7 @@ module_params(Es) ->
 %% 			     [edoc_lib:datestr(date()),
 %% 			      edoc_lib:timestr(time())])
 %% 	      ]}]}].
- 
+
 stylesheet(Opts) ->
     case Opts#opts.stylesheet of
 	undefined ->
@@ -479,7 +479,7 @@ label_anchor(Content, E) ->
 
 %% This is currently only done for functions without type spec.
 
-signature(Es, Name) -> 
+signature(Es, Name) ->
     [{tt, [Name, "("] ++ seq(fun arg/1, Es) ++ [") -> any()"]}].
 
 arg(#xmlElement{content = Es}) ->
@@ -605,7 +605,7 @@ pp_clause(Pre, Type) ->
     L1 = erl_pp:attribute({attribute,0,spec,{{list_to_atom(Atom),0},[Types]}}),
     "-spec " ++ L2 = lists:flatten(L1),
     L3 = Pre ++ lists:nthtail(length(Atom), L2),
-    re:replace(L3, "\n      ", "\n", [{return,list},global]).
+    re:replace(L3, "\n      ", "\n", [{return,list},global,unicode]).
 
 format_type(Prefix, Name, Type, Last, #opts{pretty_printer = erl_pp}=Opts) ->
     try
@@ -628,7 +628,7 @@ pp_type(Prefix, Type) ->
                  "::\n" ++ L3 -> {"\n"++L3,6}
              end,
     Ss = lists:duplicate(N, $\s),
-    re:replace(L2, "\n"++Ss, "\n", [{return,list},global]).
+    re:replace(L2, "\n"++Ss, "\n", [{return,list},global,unicode]).
 
 etypef(L, O0) ->
     {R, O} = etypef(L, [], O0, []),
@@ -1253,7 +1253,7 @@ get_first_sentence(Es) ->
 
 get_first_sentence_1(Es) ->
     get_first_sentence_1(Es, []).
-    
+
 get_first_sentence_1([E = #xmlText{value = Txt} | Es], Acc) ->
     Last = case Es of
 	       [#xmlElement{name = p} | _] -> true;

src/edown_lib.erl

@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%==============================================================================
 %% @author Ulf Wiger <[email protected]>
-%% @copyright 2010 Erlang Solutions Ltd 
+%% @copyright 2010 Erlang Solutions Ltd
 %% @end
 %% =====================================================================
 
@@ -65,7 +65,7 @@ redirect_uri(Href, _Name, E) ->
     case lists:member("/", Href) of
 	false ->
 	    [_|_] = URI = get_attrval(href, E),
-	    NewURI = re:replace(URI,".html",".md",[{return,list}]),
+	    NewURI = re:replace(URI,".html",".md",[{return,list},unicode]),
 	    replace_uri(NewURI, E);
 	true ->
 	    false

src/edown_xmerl.erl

@@ -46,14 +46,15 @@
     brstrip(to_string(Text)).
 
 to_string(S) ->
-    binary_to_list(iolist_to_binary([S])).
+    %%binary_to_list(iolist_to_binary([S])).
+    unicode:characters_to_list([S]).
 
 strip(Str) -> lstrip(rstrip(Str)).
-lstrip(Str) -> re:replace(Str,"^\\s","",[]).
-rstrip(Str) -> re:replace(Str, "\\s\$", []).
+lstrip(Str) -> re:replace(Str,"^\\s","", [unicode]).
+rstrip(Str) -> re:replace(Str, "\\s\$", "", [unicode]).
 
 % Strip double spaces at end of line -- markdown reads as hard return.
-brstrip(Str) -> re:replace(Str, "\\s+\\s\$", "", [global, multiline]).
+brstrip(Str) -> re:replace(Str, "\\s+\\s\$", "", [global, multiline, unicode]).
 
 %% The '#root#' tag is called when the entire structure has been
 %% exported. It does not appear in the structure itself.
@@ -109,7 +110,7 @@ elem(Tag, Data, Attrs, Parents, E) ->
     end.
 
 escape_pre(Data) ->
-    re:replace(re:replace(Data, "<", "\\&lt;", [global]), ">", "\\&gt;", [global]).
+    re:replace(re:replace(Data, "<", "\\&lt;", [global, unicode]), ">", "\\&gt;", [global, unicode]).
 
 %% Given content of a pre tag in `Data', entity escape angle brackets
 %% but leave anchor tags alone. This is less than pretty, but is