introduce new encoding concept by default, store data in utf-8, local encodings only in pages

git-svn-id: https://webim.svn.sourceforge.net/svnroot/webim/trunk@64 c66351dc-e62f-0410-b875-e3a5c0b9693f

src/webim/install/index.php

@@ -32,7 +32,7 @@ function check_connection() {
 	if ($link) {
 		$result = mysql_query("SELECT VERSION() as c", $link);
 		if( $result && $ver = mysql_fetch_array($result, MYSQL_ASSOC)) {
-			$page['done'][] = getstring2("install.1.connected", array($ver['c']));
+			$page['done'][] = getlocal2("install.1.connected", array($ver['c']));
 			mysql_free_result($result);
 		} else {
 			$errors[] = "Version of your SQL server is unknown. Please check. Error: ".mysql_error();
@@ -41,7 +41,7 @@ function check_connection() {
 		}
 		return $link;
 	} else {
-		$errors[] = getstring2("install.connection.error", array(mysql_error()));
+		$errors[] = getlocal2("install.connection.error", array(mysql_error()));
 		return null;
 	}
 }
@@ -49,14 +49,14 @@ function check_connection() {
 function check_database($link) {
 	global $mysqldb, $force_charset_in_connection, $dbencoding, $page;
 	if(mysql_select_db($mysqldb,$link)) {
-		$page['done'][] = getstring2("install.2.db_exists", array($mysqldb));
+		$page['done'][] = getlocal2("install.2.db_exists", array($mysqldb));
 		if( $force_charset_in_connection ) {
 			mysql_query("SET character set $dbencoding", $link);
 		}
 		return true;
 	} else {
-		$page['nextstep'] = getstring2("install.2.create", array($mysqldb));
+		$page['nextstep'] = getlocal2("install.2.create", array($mysqldb));
-		$page['nextnotice'] = getstring("install.2.notice");
+		$page['nextnotice'] = getlocal("install.2.notice");
 		$page['nextstepurl'] = "dbperform.php?act=createdb";
 	}
 	return false;
@@ -68,10 +68,10 @@ function check_tables($link) {
 	if( $curr_tables !== false) {
 		$tocreate = array_diff(array_keys($dbtables), $curr_tables);
 		if( count($tocreate) == 0 ) {
-			$page['done'][] = getstring("install.3.tables_exist");
+			$page['done'][] = getlocal("install.3.tables_exist");
 			return true;
 		} else {
-			$page['nextstep'] = getstring("install.3.create");
+			$page['nextstep'] = getlocal("install.3.create");
 			$page['nextstepurl'] = "dbperform.php?act=createtables";
 		}
 	}
@@ -92,9 +92,9 @@ function check_columns($link) {
 			$cannot_update = array_diff($tocreate, $dbtables_can_update[$id]);
 			if( count($cannot_update) != 0) {
 				$errors[] = "Key columns are absent in table `$id'. Unable to continue installation.";
-				$page['nextstep'] = getstring("install.kill_tables");
+				$page['nextstep'] = getlocal("install.kill_tables");
 				$page['nextstepurl'] = "dbperform.php?act=droptables";
-				$page['nextnotice'] = getstring("install.kill_tables.notice");
+				$page['nextnotice'] = getlocal("install.kill_tables.notice");
 				return false;
 			}
 			$need_to_create_columns = true;
@@ -102,13 +102,13 @@ function check_columns($link) {
 	}
 
 	if( $need_to_create_columns ) {
-		$page['nextstep'] = getstring("install.4.create");
+		$page['nextstep'] = getlocal("install.4.create");
 		$page['nextstepurl'] = "dbperform.php?act=addcolumns";
-		$page['nextnotice'] = getstring("install.4.notice");
+		$page['nextnotice'] = getlocal("install.4.notice");
 		return false;
 	}
 
-	$page['done'][] = getstring("install.4.done");
+	$page['done'][] = getlocal("install.4.done");
 	return true;
 }
 
@@ -134,10 +134,10 @@ function check_status() {
 		return;
 	}
 
-	$page['done'][] = getstring("installed.message");
+	$page['done'][] = getlocal("installed.message");
 
-	$page['nextstep'] = getstring("installed.login_link");
+	$page['nextstep'] = getlocal("installed.login_link");
-	$page['nextnotice'] = getstring("installed.notice");
+	$page['nextnotice'] = getlocal("installed.notice");
 	$page['nextstepurl'] = "$webimroot/";
 
 	mysql_close($link);

src/webim/leavemessage.php

@@ -35,14 +35,16 @@ if( !$email ) {
 }
 
 if( count($errors) > 0 ) {
-	$page['formname'] = $visitor_name;
+	$page['formname'] = topage($visitor_name);
 	$page['formemail'] = $email;
-	$page['formmessage'] = $message;
+	$page['formmessage'] = topage($message);
 	start_html_output();
 	require('view/chat_leavemsg.php');
 	exit;
 }
 
+// FIXME mail encoding
+
 $subject = getstring2_("leavemail.subject", array($visitor_name), $webim_messages_encoding);
 $body = getstring2_("leavemail.body", array($visitor_name,$email,$message), $webim_messages_encoding); 
 

src/webim/libs/chat.php

@@ -114,7 +114,7 @@ function get_messages($threadid,$meth,$isuser,&$lastid) {
         if ($meth == 'xml') {
             $message = "<message>".myiconv($webim_encoding,"utf-8",escape_with_cdata(message_to_html($msg)))."</message>\n";
         } else {
-            $message = (($meth == 'text') ? message_to_text($msg) : message_to_html($msg));
+            $message = (($meth == 'text') ? message_to_text($msg) : topage(message_to_html($msg)));
         }
 
 		$messages[] = $message;
@@ -181,11 +181,11 @@ function setup_chatview_for_user($thread,$level) {
 	$page['level'] = $level;
 	$page['ct.chatThreadId'] = $thread['threadid'];
 	$page['ct.token'] = $thread['ltoken'];
-	$page['ct.user.name'] = $thread['userName'];
+	$page['ct.user.name'] = topage($thread['userName']);
 	$page['canChangeName'] = $user_can_change_name;
 
-	$page['ct.company.name'] = $company_name;
+	$page['ct.company.name'] = topage($company_name);
-	$page['ct.company.chatLogoURL'] = $company_logo_link;
+	$page['ct.company.chatLogoURL'] = topage($company_logo_link);
 	$page['send_shortcut'] = "Ctrl-Enter";
 
 	$params = "thread=".$thread['threadid']."&token=".$thread['ltoken'];
@@ -201,10 +201,10 @@ function setup_chatview_for_operator($thread,$operator) {
 	$page['canpost'] = true;
 	$page['ct.chatThreadId'] = $thread['threadid'];
 	$page['ct.token'] = $thread['ltoken'];
-	$page['ct.user.name'] = get_user_name($thread['userName']);
+	$page['ct.user.name'] = topage(get_user_name($thread['userName']));
 
-	$page['ct.company.name'] = $company_name;
+	$page['ct.company.name'] = topage($company_name);
-	$page['ct.company.chatLogoURL'] = $company_logo_link;
+	$page['ct.company.chatLogoURL'] = topage($company_logo_link);
 	$page['send_shortcut'] = "Ctrl-Enter";
 
 	// TODO

src/webim/libs/common.php

@@ -21,10 +21,14 @@ $version = 'v1.0.9';
 
 function myiconv($in_enc, $out_enc, $string) {
 	global $_utf8win1251, $_win1251utf8;
+	if($in_enc == $out_enc ) {
+		return $string;
+	}
 	if( function_exists('iconv') ) {
-	    $converted = iconv($in_enc, $out_enc, $string);
+	    $converted = @iconv($in_enc, $out_enc, $string);
-	    if( $converted !== FALSE )
+	    if( $converted !== FALSE ) {
 			return $converted;
+		}
 	}
 	if( $in_enc == "cp1251" && $out_enc == "utf-8" )
 		return strtr($string, $_win1251utf8);
@@ -34,21 +38,6 @@ function myiconv($in_enc, $out_enc, $string) {
 	return $string; // do not know how to convert
 }
 
-function getrawparam( $name ) {
-	global $webim_encoding, $request_encoding;
-	if( isset($_POST[$name]) )
-		return myiconv($request_encoding,$webim_encoding,$_POST[$name]);
-	die("no ".$name." parameter");
-}
-
-function getparam( $name ) {
-	if( isset($_POST[$name]) )
-		return $_POST[$name];
-    if( isset($_GET[$name]))
-        return $_GET[$name];
-	die("no ".$name." parameter");
-}
-
 function verifyparam( $name, $regexp, $default = null ) {
 
 	if( isset( $_GET[$name] ) ) {
@@ -121,6 +110,7 @@ function set_locale($locale) {
 
 $current_locale = get_locale();
 $messages = array();
+$output_encoding = array();
 
 function get_locale_links($href) {
 	global $available_locales, $current_locale;
@@ -137,20 +127,36 @@ function get_locale_links($href) {
 }
 
 function load_messages($locale) {
-	global $messages;
+	global $messages, $webim_encoding, $output_encoding;
 	$hash = array();
+	$current_encoding = $webim_encoding;
 	$fp = fopen(dirname(__FILE__)."/../locales/$locale/properties","r");
 	while (!feof($fp)) {
     	$line = fgets($fp, 4096);
 		$keyval = split("=", $line, 2 );
 		if( isset($keyval[1]) ) {
-			$hash[$keyval[0]] = str_replace("\\n", "\n",trim($keyval[1]));
+			if($keyval[0] == 'encoding') {
+				$current_encoding = trim($keyval[1]);
+			} else if($keyval[0] == 'output_encoding') {
+				$output_encoding[$locale] = trim($keyval[1]); 
+			} else if( $current_encoding == $webim_encoding ) {
+				$hash[$keyval[0]] = str_replace("\\n", "\n",trim($keyval[1]));
+			} else { 
+				$hash[$keyval[0]] = myiconv($current_encoding, $webim_encoding, str_replace("\\n", "\n",trim($keyval[1])));
+			}
 		}
 	}
 	fclose($fp);
 	$messages[$locale] = $hash;
 }
 
+function getoutputenc() {
+	global $current_locale, $output_encoding, $webim_encoding, $messages;
+	if(!isset($messages[$current_locale]))
+		load_messages($current_locale);
+	return isset($output_encoding[$current_locale]) ? $output_encoding[$current_locale] : $webim_encoding;
+}
+
 function getstring_($text,$locale) {
 	global $messages;
 	if(!isset($messages[$locale]))
@@ -168,6 +174,16 @@ function getstring($text) {
 	return getstring_($text,$current_locale);
 }
 
+function getlocal($text) {
+	global $current_locale, $webim_encoding;
+	return myiconv($webim_encoding,getoutputenc(), getstring_($text,$current_locale));
+}
+
+function topage($text) {
+	global $webim_encoding;
+	return myiconv($webim_encoding,getoutputenc(), $text);
+}
+
 function getstring2_($text,$params,$locale) {
 	$string = getstring_($text,$locale);
 	for( $i = 0; $i < count($params); $i++ ) {
@@ -181,6 +197,31 @@ function getstring2($text,$params) {
 	return getstring2_($text,$params,$current_locale);
 }
 
+function getlocal2($text,$params) {
+	global $current_locale, $webim_encoding;
+	$string = myiconv($webim_encoding,getoutputenc(), getstring_($text,$current_locale));
+	for( $i = 0; $i < count($params); $i++ ) {
+		$string = str_replace("{".$i."}", $params[$i], $string);
+	}
+	return $string;
+}
+
+/* ajax server actions use utf-8 */
+function getrawparam( $name ) {
+	global $webim_encoding;
+	if( isset($_POST[$name]) )
+		return myiconv("utf-8",$webim_encoding,$_POST[$name]);
+	die("no ".$name." parameter");
+}
+
+/* form processors use current Output encoding */
+function getparam( $name ) {
+	global $webim_encoding;
+	if( isset($_POST[$name]) )
+		return myiconv(getoutputenc(), $webim_encoding, $_POST[$name]);
+	die("no ".$name." parameter");
+}
+
 function connect() {
 	global $mysqlhost, $mysqllogin, $mysqlpass, $mysqldb, $dbencoding, $force_charset_in_connection;
 	$link = @mysql_connect($mysqlhost,$mysqllogin ,$mysqlpass )
@@ -224,9 +265,9 @@ function start_xml_output() {
 }
 
 function start_html_output() {
-	global $output_charset;
+	$charset = getstring("output_charset");
 	header("Cache-Control: no-store, no-cache, must-revalidate");
-	header("Content-type: text/html".(isset($output_charset)?"; charset=".$output_charset:""));
+	header("Content-type: text/html".(isset($charset)?"; charset=".$charset:""));
 }
 
 function escape_with_cdata($text) {
@@ -248,12 +289,12 @@ function form_value_cb($key) {
 }
 
 function no_field($key) {
-	return getstring2("errors.required",array(getstring($key)));
+	return getlocal2("errors.required",array(getlocal($key)));
 }
 
 
 function wrong_field($key) {
-	return getstring2("errors.wrong_field",array(getstring($key)));
+	return getlocal2("errors.wrong_field",array(getlocal($key)));
 }
 
 function get_popup($href,$message,$title,$wndName,$options) {

src/webim/libs/config.php

@@ -31,22 +31,17 @@ $mysqlpass = "123";
 
 
 /* Use CP-1251 database */
+/*
 $dbencoding = "cp1251";
 $webim_encoding = "cp1251";
-$request_encoding = "utf-8";
-$output_charset = "Windows-1251";
-$force_charset_in_connection = true;
-
-
-/* Use UTF-8 database */
-/*
-$dbencoding = "utf8";
-$webim_encoding = "cp1251";
-$request_encoding = "utf-8";
-$output_charset = "Windows-1251";
 $force_charset_in_connection = true;
 */
 
+/* Use UTF-8 database */
+$dbencoding = "utf8";
+$webim_encoding = "utf-8";
+$force_charset_in_connection = true;
+
 /*
  *   From field in outgoing mail.
  */

src/webim/libs/pagination.php

@@ -63,7 +63,7 @@ function setup_empty_pagination() {
 
 function generate_pagination($pagination) {
 	global $pagination_spacing, $links_on_page;
-	$result = getstring2("tag.pagination.info",
+	$result = getlocal2("tag.pagination.info",
 		array($pagination['page'],$pagination['total'],$pagination['start']+1,$pagination['end'],$pagination['count']))."<br/>";
 
 	if( $pagination['total'] > 1 ) {

src/webim/locales/en/properties

@@ -1,3 +1,6 @@
+encoding=utf-8
+output_charset=utf-8
+output_encoding=utf-8
 site.url=http://webim.ru
 site.title=webim.ru
 company.title=Web Messenger Community

src/webim/locales/ru/properties

@@ -1,3 +1,6 @@
+encoding=cp1251
+output_charset=Windows-1251
+output_encoding=cp1251
 site.url=http://webim.ru
 site.title=webim.ru
 company.title=Web Messenger Community

src/webim/mail.php

@@ -52,6 +52,8 @@ foreach( $output as $msg ) {
 	$history .= $msg;
 }
 
+// FIXME mail encoding?
+
 $subject = getstring("mail.user.history.subject");
 $body = getstring2("mail.user.history.body", array($thread['userName'],$history) ); 
 

src/webim/operator/getcode.php

@@ -52,7 +52,7 @@ $size = get_gifimage_size($file);
 $message = get_image(get_app_location($showhost,$forcesecure)."/button.php?image=$image&lang=$lang",$size[0],$size[1]);
 
 $page = array();
-$page['operator'] = get_operator_name($operator);
+$page['operator'] = topage(get_operator_name($operator));
 $page['buttonCode'] = generate_button("",$lang,$message,$showhost,$forcesecure);
 $page['availableImages'] = array_keys($imageLocales);
 $page['availableLocales'] = $image_locales;

src/webim/operator/history.php

@@ -19,8 +19,8 @@ require('../libs/pagination.php');
 
 $operator = check_login();
 
-$page = array( 'operator' => get_operator_name($operator) );
+$page = array( 'operator' => topage(get_operator_name($operator)) );
-$query = isset($_GET['q']) ? $_GET['q'] : false;
+$query = isset($_GET['q']) ? myiconv(getoutputenc(), $webim_encoding, $_GET['q']) : false;
 
 if($query !== false) {
 	$link = connect();
@@ -44,7 +44,7 @@ if($query !== false) {
 	mysql_free_result($result);
 	mysql_close($link);
 
-	$page['formq'] = $query;
+	$page['formq'] = topage($query);
 	setup_pagination($foundThreads);
 } else {
 	setup_empty_pagination();

src/webim/operator/index.php

@@ -18,7 +18,7 @@ require('../libs/operator.php');
 $operator = check_login();
 
 $page = array( 
-	'operator' => get_operator_name($operator),
+	'operator' => topage(get_operator_name($operator)),
 	'version' => $version,
 	'localeLinks' => get_locale_links("$webimroot/operator/index.php")
 );

src/webim/operator/login.php

@@ -32,7 +32,7 @@ if( isset($_POST['login']) && isset($_POST['password']) ) {
 		header("Location: $target");
 		exit;
 	} else {
-		$errors[] = getstring("page_login.error");
+		$errors[] = getlocal("page_login.error");
 	}
 }
 

src/webim/operator/operator.php

@@ -36,6 +36,7 @@ if( isset($_POST['login']) && isset($_POST['password']) ) {
 	if( !$commonname )
 		$errors[] = no_field("form.field.agent_commonname");
 
+	// FIXME check login \w+
 	if( !$login )
 		$errors[] = no_field("form.field.login");
 
@@ -43,12 +44,12 @@ if( isset($_POST['login']) && isset($_POST['password']) ) {
 		$errors[] = no_field("form.field.password");
 
 	if( $password != $passwordConfirm )
-		$errors[] = getstring("my_settings.error.password_match");
+		$errors[] = getlocal("my_settings.error.password_match");
 
 	$existing_operator = operator_by_login($login);
 	if( (!$agentId && $existing_operator) || 
 		( $agentId && $existing_operator && $agentId != $existing_operator['operatorid']) )
-		$errors[] = getstring("page_agent.error.duplicate_login");
+		$errors[] = getlocal("page_agent.error.duplicate_login");
 
 	if( count($errors) == 0 ) {
 		if (!$agentId) {
@@ -59,10 +60,10 @@ if( isset($_POST['login']) && isset($_POST['password']) ) {
 		header("Location: ".dirname($_SERVER['PHP_SELF'])."/operators.php");
 		exit;
 	} else {
-		$page['formlogin'] = $login;
+		$page['formlogin'] = topage($login);
-		$page['formname'] = $localname;
+		$page['formname'] = topage($localname);
-		$page['formcommonname'] = $commonname;
+		$page['formcommonname'] = topage($commonname);
-		$page['agentId'] = $agentId;
+		$page['agentId'] = topage($agentId);
 	}
 
 } else if( isset($_GET['op']) ) {
@@ -70,17 +71,17 @@ if( isset($_POST['login']) && isset($_POST['password']) ) {
 	$op = operator_by_login( $login );
 
 	if( !$op ) {
-		$errors[] = getstring("no_such_operator");
+		$errors[] = getlocal("no_such_operator");
-		$page['formlogin'] = $login;
+		$page['formlogin'] = topage($login);
 	} else {
-		$page['formlogin'] = $op['vclogin'];
+		$page['formlogin'] = topage($op['vclogin']);
-		$page['formname'] = $op['vclocalename'];
+		$page['formname'] = topage($op['vclocalename']);
-		$page['formcommonname'] = $op['vccommonname'];
+		$page['formcommonname'] = topage($op['vccommonname']);
-		$page['agentId'] = $op['operatorid'];
+		$page['agentId'] = topage($op['operatorid']);
 	}
 }
 
-$page['operator'] = get_operator_name($operator);
+$page['operator'] = topage(get_operator_name($operator));
 
 start_html_output();
 require('../view/agent.php');

src/webim/operator/operators.php

@@ -19,7 +19,7 @@ $operator = check_login();
 
 $page = array();
 $page['allowedAgents'] = get_operators();
-$page['operator'] = get_operator_name($operator);
+$page['operator'] = topage(get_operator_name($operator));
 
 start_html_output();
 require('../view/agents.php');

src/webim/operator/threadprocessor.php

@@ -18,7 +18,7 @@ require('../libs/chat.php');
 
 $operator = check_login();
 
-$page = array( 'operator' => get_operator_name($operator) );
+$page = array( 'operator' => topage(get_operator_name($operator)) );
 
 
 if( isset($_GET['threadid'])) {

src/webim/operator/update.php

@@ -38,13 +38,13 @@ $threadstate_key = array(
 );
 
 function thread_to_xml($thread) {
-	global $threadstate_to_string, $threadstate_key;
+	global $threadstate_to_string, $threadstate_key, $webim_encoding;
 	$state = $threadstate_to_string[$thread['istate']];
 	$result = "<thread id=\"".$thread['threadid']."\" stateid=\"$state\"";
 	if( $state == "closed" )
 		return $result."/>";
 
-	$state = getstring($threadstate_key[$thread['istate']]);
+	$state = myiconv($webim_encoding,"utf-8",getstring($threadstate_key[$thread['istate']]));
 	$threadoperator = ($thread['agentName'] ? $thread['agentName'] : "-");
 
 	$result .= " canopen=\"true\"";

src/webim/operator/users.php

@@ -20,7 +20,7 @@ $operator = check_login();
 
 notify_operator_alive($operator['operatorid']);
 
-$page = array( 'operator' => get_operator_name($operator) );
+$page = array( 'operator' => topage(get_operator_name($operator)) );
 
 start_html_output();
 require('../view/pending_users.php');