From 248847436b32f685cfbbdb901edbe56ede19aa8b Mon Sep 17 00:00:00 2001 From: Dmitriy Simushev Date: Wed, 19 Jun 2013 09:15:29 +0000 Subject: [PATCH] Remove messages indexing --- src/messenger/webim/cron.php | 2 - src/messenger/webim/install/dbinfo.php | 21 +- src/messenger/webim/install/dbperform.php | 4 - src/messenger/webim/libs/classes/thread.php | 8 +- src/messenger/webim/libs/cron.php | 193 ------------------ src/messenger/webim/libs/statistics.php | 8 +- src/messenger/webim/locales/en/properties | 2 +- src/messenger/webim/locales/ru/properties | 2 +- src/messenger/webim/operator/history.php | 10 +- .../webim/operator/threadprocessor.php | 2 +- src/messenger/webim/view/thread_search.php | 2 +- 11 files changed, 15 insertions(+), 239 deletions(-) diff --git a/src/messenger/webim/cron.php b/src/messenger/webim/cron.php index b18ecca3..b8385dc1 100644 --- a/src/messenger/webim/cron.php +++ b/src/messenger/webim/cron.php @@ -35,8 +35,6 @@ $quiet = isset($_GET['q']); set_time_limit(0); // Run cron jobs of the core -cron_index_messages(); - calculate_thread_statistics(); calculate_operator_statistics(); calculate_page_statistics(); diff --git a/src/messenger/webim/install/dbinfo.php b/src/messenger/webim/install/dbinfo.php index 8d5d6e70..3aaed7c8 100644 --- a/src/messenger/webim/install/dbinfo.php +++ b/src/messenger/webim/install/dbinfo.php @@ -87,10 +87,7 @@ $dbtables = array( "arguments" => "varchar(1024)" ), - // Temporal message storage. Contain only messages for for open threads. - // There is cron job that clean up this table and move messages of the core - // kinds to ${mysqlprefix}indexedchatmessage. Plugins messages must be - // converted to one of the core kinds before move. + // Store chat thread messages "${mysqlprefix}chatmessage" => array( "messageid" => "int NOT NULL auto_increment PRIMARY KEY", "threadid" => "int NOT NULL references ${mysqlprefix}chatthread(threadid)", @@ -101,18 +98,6 @@ $dbtables = array( "tname" => "varchar(64)" ), - // Indexed messages used for search, history and statistics. - // Contain messages only of the Core kinds. - "${mysqlprefix}indexedchatmessage" => array( - "messageid" => "int NOT NULL auto_increment PRIMARY KEY", - "threadid" => "int NOT NULL references ${mysqlprefix}chatthread(threadid)", - "ikind" => "int NOT NULL", - "agentId" => "int NOT NULL DEFAULT 0", - "tmessage" => "text NOT NULL", - "dtmcreated" => "int NOT NULL DEFAULT 0", - "tname" => "varchar(64)" - ), - "${mysqlprefix}chatoperator" => array( "operatorid" => "int NOT NULL auto_increment PRIMARY KEY", "vclogin" => "varchar(64) NOT NULL", @@ -229,9 +214,6 @@ $dbtables_indexes = array( "${mysqlprefix}chatmessage" => array( "idx_agentid" => "agentid" ), - "${mysqlprefix}indexedchatmessage" => array( - "agentid" => "agentid" - ), "${mysqlprefix}chatsitevisitor" => array( "threadid" => "threadid" ), @@ -250,7 +232,6 @@ $dbtables_can_update = array( "${mysqlprefix}chatthreadstatistics" => array("missedthreads", "sentinvitations", "acceptedinvitations", "rejectedinvitations", "ignoredinvitations"), "${mysqlprefix}requestbuffer" => array("requestid", "requestkey", "request"), "${mysqlprefix}chatmessage" => array("agentId"), - "${mysqlprefix}indexedchatmessage" => array(), "${mysqlprefix}chatoperator" => array("vcavatar", "vcjabbername", "iperm", "istatus", "idisabled", "vcemail", "dtmrestore", "vcrestoretoken", "code"), "${mysqlprefix}chatoperatorstatistics" => array("sentinvitations", "acceptedinvitations", "rejectedinvitations", "ignoredinvitations"), "${mysqlprefix}chatban" => array(), diff --git a/src/messenger/webim/install/dbperform.php b/src/messenger/webim/install/dbperform.php index db3a4338..5aa1e8d8 100644 --- a/src/messenger/webim/install/dbperform.php +++ b/src/messenger/webim/install/dbperform.php @@ -289,10 +289,6 @@ if ($act == "silentcreateall") { runsql("ALTER TABLE ${mysqlprefix}chatmessage ADD INDEX idx_agentid (agentid)", $link); } - if (in_array("${mysqlprefix}indexedchatmessage.agentid", $absent_indexes)) { - runsql("ALTER TABLE ${mysqlprefix}indexedchatmessage ADD INDEX (agentid)", $link); - } - if (in_array("${mysqlprefix}chatsitevisitor.threadid", $absent_indexes)) { runsql("ALTER TABLE ${mysqlprefix}chatsitevisitor ADD INDEX (threadid)", $link); } diff --git a/src/messenger/webim/libs/classes/thread.php b/src/messenger/webim/libs/classes/thread.php index f1b38b8d..91a5e04c 100644 --- a/src/messenger/webim/libs/classes/thread.php +++ b/src/messenger/webim/libs/classes/thread.php @@ -684,8 +684,6 @@ Class Thread { * @param boolean $is_user Boolean TRUE if messages loads for user * and boolean FALSE if they loads for operator. * @param int $lastid ID of the last loaded message. - * @param boolean $indexed Indicates if indexed messages be should returned. - * Default value is false. * @return array Array of messages. Every message is associative array with * following keys: * - 'id': int, message id; @@ -696,18 +694,16 @@ Class Thread { * Thread::KIND_PLUGIN and message text string otherwise. * @see Thread::postMessage() */ - public function getMessages($is_user, &$last_id, $indexed = false) { + public function getMessages($is_user, &$last_id) { global $webim_encoding; $db = Database::getInstance(); - $messages_table = $indexed ? '{indexedchatmessage}' : '{chatmessage}'; - // Load messages $messages = $db->query( "select messageid as id, ikind as kind, dtmcreated as created, " . " tname as name, tmessage as message " . - "from " . $messages_table . " " . + "from {chatmessage} " . "where threadid = :threadid and messageid > :lastid " . ($is_user ? "and ikind <> " . self::KIND_FOR_AGENT : "") . " order by messageid", diff --git a/src/messenger/webim/libs/cron.php b/src/messenger/webim/libs/cron.php index e9c25e4f..eef82b0a 100644 --- a/src/messenger/webim/libs/cron.php +++ b/src/messenger/webim/libs/cron.php @@ -15,199 +15,6 @@ * limitations under the License. */ -/** - * Index messages for closed threads - * - * History and serarch works only for indexed messages. - * - * Trigger 'pluginMessageIndex' event. Plugins can use this event to - * index their own messages. - * - * Event listener receives as argument an associative array with following keys: - * - 'message': associative array with message data; - * - 'result': array of indexed message data. Default value is boolean false. - * - * The 'message' element contains following keys: - * - 'plugin': string, name of the plugin which sent the message; - * - 'data': array, data sent by the plugin. - * - * Plugin can use the 'result' element to return indexed message data. By default it - * equals to boolean false. To index message plugin should set this element to - * an associative array with following keys: - * - 'kind': int, indexed message kind. It must be one of the core messages - * kinds, not Thread::KIND_PLUGIN. - * - 'message': string, text of indexed message. - * - 'sender_name': string, name of person who sent the message. This field is - * arbitrary and do not use for some messages kinds. - * - 'operator_id': int, ID of the operator who send the message. This field is - * arbitrary and do not use for some messages kinds. - * - * If the 'result' element equals to boolean false message will be skipped. - * - * Example of event listener is listed below: - * - * public function pluginMessageIndexListener(&$args) { - * // Create shortcut for stored data - * $data = $args['message']['data']; - * // Check if message was sent by current plugin - * if ($args['message']['plugin'] == 'example') { - * $args['result'] = array( - * // Plugin should set one of the core message kinds to indexed message - * 'kind' => Thread::KIND_INFO, - * // Plugin should set arbitrary text for indexed message - * 'message' => $data['title'] . ': ' . $data['body'] - * ); - * } - * } - * - */ -function cron_index_messages() { - $db = Database::getInstance(); - $db_throw_exceptions = $db->throwExeptions(true); - - try { - // Start transaction - $db->query('START TRANSACTION'); - - // Select messages from closed threads - $messages = $db->query( - "SELECT {chatmessage}.* FROM {chatmessage}, {chatthread} " . - "WHERE {chatmessage}.threadid = {chatthread}.threadid AND " . - "({chatthread}.istate = :closed OR {chatthread}.istate = :left)", - array( - ':closed' => Thread::STATE_CLOSED, - ':left' => Thread::STATE_LEFT - ), - array('return_rows' => Database::RETURN_ALL_ROWS) - ); - - // Prevent race condition. Remove only moved to {indexedchatmessage} - // messages. Get last loaded message ID for that. - $last_id = 0; - $dispatcher = EventDispatcher::getInstance(); - foreach($messages as $key => $message) { - // Find last message ID - if ($message['messageid'] > $last_id) { - $last_id = $message['messageid']; - } - - // Leave core messages kind as is - if ($message['ikind'] != Thread::KIND_PLUGIN) { - continue; - } - - // Provide an ability for plugins to index own messages - $event_args = array( - 'message' => unserialize($message['tmessage']), - 'result' => false - ); - $dispatcher->triggerEvent('pluginMessageIndex', $event_args); - - // Check if message was processed by a plugin correctly - $update_message = true; - if (empty($event_args['result']['message'])) { - $update_message = false; - } - - // Check if kind set and correct - if (empty($event_args['result']['kind']) - || $event_args['result']['kind'] == Thread::KIND_PLUGIN) { - $update_message = false; - } - - // Check if message should be updated - if (! $update_message) { - unset($messages[$key]); - continue; - } - - // Update message - $messages[$key]['ikind'] = $event_args['result']['kind']; - $messages[$key]['tmessage'] = $event_args['result']['message']; - - if (array_key_exists('sender_name', $event_args['result'])) { - $messages[$key]['tname'] = $event_args['result']['sender_name']; - } - - if (array_key_exists('operator_id', $event_args['result'])) { - $messages[$key]['agentId'] = $event_args['result']['operator_id']; - } - } - - // Check is there some messages that should be saved - if (count($messages) != 0) { - // Reindex messages array - $messages = array_values($messages); - // Prepare SQL query template - $message_fields = array_keys($messages[0]); - $placeholders = '(' . - implode(', ', array_fill(0, count($message_fields), '?')) . ')'; - $sql_template = 'INSERT INTO {indexedchatmessage} (' . - implode(', ', $message_fields) . ') VALUES '; - - // Insert indexed messages into database by $block_size messages per - // sql query - $block_size = 20; - $iteration_count = ceil(count($messages) / $block_size); - for($i = 0; $i < $iteration_count; $i++) { - // Get messages block - $messages_block = array_slice( - $messages, - $i * $block_size, - $block_size - ); - - // Count of $messages_block can be less than $block_size for - // the last block of messages. - $real_block_size = count($messages_block); - - // Build array of inserted values - $fields_to_insert = array(); - foreach($messages_block as $message) { - foreach($message_fields as $field_name) { - $fields_to_insert[] = $message[$field_name]; - } - } - - // Build query - $sql = $sql_template . implode( - ', ', - array_fill(0, $real_block_size, $placeholders) - ); - - // Run query - $db->query($sql, $fields_to_insert); - } - } - - // Check is there some processed messages that should be deleted - if ($last_id != 0) { - // Delete indexed messages - $db->query( - 'DELETE FROM {chatmessage} where messageid <= :last_id', - array(':last_id' => $last_id) - ); - } - } catch (Exception $e) { - // Something went wrong: warn and rollback transaction. - trigger_error( - 'Messages indexing faild: ' . $e->getMessage(), - E_USER_WARNING - ); - $db->query('ROLLBACK'); - - // Set throw exceptions back - $db->throwExeptions($db_throw_exceptions); - return; - } - - // Commit transaction - $db->query('COMMIT'); - - // Set throw exceptions back - $db->throwExeptions($db_throw_exceptions); -} - /** * Generates cron URI * diff --git a/src/messenger/webim/libs/statistics.php b/src/messenger/webim/libs/statistics.php index 9bcfc8b3..522ec714 100644 --- a/src/messenger/webim/libs/statistics.php +++ b/src/messenger/webim/libs/statistics.php @@ -82,7 +82,7 @@ function calculate_thread_statistics() { "SUM(m.ikind = :kind_user) AS user_msgs, " . "MAX(m.dtmcreated) as last_msg_time, " . "threadid " . - "FROM {indexedchatmessage} m " . + "FROM {chatmessage} m " . // Calculate only users' and operators' messages "WHERE m.ikind = :kind_user " . "OR m.ikind = :kind_agent " . @@ -293,10 +293,10 @@ function calculate_operator_statistics() { "COUNT(distinct m.threadid) AS threads, " . "COUNT(m.messageid) AS messages, " . "AVG(CHAR_LENGTH(m.tmessage)) AS avg_msg_length " . - // Use {indexedchatmessage} as base table because of one thread can + // Use {chatmessage} as base table because of one thread can // be related with more than one operator (they can change each // other during conversation). - "FROM {indexedchatmessage} m, {chatthread} t " . + "FROM {chatmessage} m, {chatthread} t " . "WHERE m.ikind = :kind_agent " . "AND m.threadid = t.threadid " . "AND (m.dtmcreated - :start) > 24*60*60 " . @@ -472,7 +472,7 @@ function calculate_page_statistics() { "(SELECT " . "COUNT(*) AS msgs, " . "m.threadid " . - "FROM {indexedchatmessage} m " . + "FROM {chatmessage} m " . "WHERE m.ikind = :kind_user OR m.ikind = :kind_agent " . "GROUP BY m.threadid) tmp " . "WHERE t.referer = p.address " . diff --git a/src/messenger/webim/locales/en/properties b/src/messenger/webim/locales/en/properties index ac25d58b..bec09c18 100644 --- a/src/messenger/webim/locales/en/properties +++ b/src/messenger/webim/locales/en/properties @@ -404,7 +404,7 @@ page_login.login=Login: page_login.password=Password: page_login.remember=Remember page_login.title=Login -page_search.intro=Search the chat history for a specified user, an operator or a specified phrase in messages. Last time messages was indexed {0}. You can index them manually. +page_search.intro=Search the chat history for a specified user, an operator or a specified phrase in messages. page_search.type.all=everywhere page_search.search.type.in_system_messages=Search in system messages page_search.type.message=in messages diff --git a/src/messenger/webim/locales/ru/properties b/src/messenger/webim/locales/ru/properties index 37303295..42322d02 100644 --- a/src/messenger/webim/locales/ru/properties +++ b/src/messenger/webim/locales/ru/properties @@ -404,7 +404,7 @@ page_login.operator.disabled= page_login.password=Пароль: page_login.remember=Запомнить page_login.title=Вход в систему -page_search.intro=На данной странице можно осуществить поиск диалогов по имени пользователя, имени оператора или фразе, встречающейся в сообщении. Последний раз сообщения индексировались {0}. Вы можете выполнить индексацию вручную. +page_search.intro=На данной странице можно осуществить поиск диалогов по имени пользователя, имени оператора или фразе, встречающейся в сообщении. page_search.type.all=везде page_search.search.type.in_system_messages=Искать в системных сообщениях page_search.type.message=в сообщениях diff --git a/src/messenger/webim/operator/history.php b/src/messenger/webim/operator/history.php index 2523d2f8..1c3db690 100644 --- a/src/messenger/webim/operator/history.php +++ b/src/messenger/webim/operator/history.php @@ -56,8 +56,8 @@ if ($query !== false) { $searchConditions = array(); if ($searchType == 'message' || $searchType == 'all') { - $searchConditions[] = "({indexedchatmessage}.tmessage LIKE :query" . - ($searchInSystemMessages?'':" AND ({indexedchatmessage}.ikind = :kind_user OR {indexedchatmessage}.ikind = :kind_agent)") . + $searchConditions[] = "({chatmessage}.tmessage LIKE :query" . + ($searchInSystemMessages?'':" AND ({chatmessage}.ikind = :kind_user OR {chatmessage}.ikind = :kind_agent)") . ")"; if (! $searchInSystemMessages) { $values[':kind_user'] = Thread::KIND_USER; @@ -74,9 +74,9 @@ if ($query !== false) { // Load threads select_with_pagintation("DISTINCT {chatthread}.*", - "{chatthread}, {indexedchatmessage}", + "{chatthread}, {chatmessage}", array( - "{indexedchatmessage}.threadid = {chatthread}.threadid", + "{chatmessage}.threadid = {chatthread}.threadid", "({chatthread}.invitationstate = :invitation_accepted " . "OR {chatthread}.invitationstate = :invitation_not_invited)", "(" . implode(' or ', $searchConditions) . ")" @@ -96,8 +96,6 @@ if ($query !== false) { $page['formtype'] = $searchType; $page['forminsystemmessages'] = $searchInSystemMessages; -$page['cron_path'] = cron_get_uri(Settings::get('cron_key')); -$page['last_cron_run'] = Settings::get('_last_cron_run'); prepare_menu($operator); start_html_output(); diff --git a/src/messenger/webim/operator/threadprocessor.php b/src/messenger/webim/operator/threadprocessor.php index 5e301276..e3fd2272 100644 --- a/src/messenger/webim/operator/threadprocessor.php +++ b/src/messenger/webim/operator/threadprocessor.php @@ -52,7 +52,7 @@ if (isset($_GET['threadid'])) { // Build messages list $lastid = -1; - $messages = $thread_info['thread']->getMessages(false, $lastid, true); + $messages = $thread_info['thread']->getMessages(false, $lastid); $page['threadMessages'] = json_encode($messages); } diff --git a/src/messenger/webim/view/thread_search.php b/src/messenger/webim/view/thread_search.php index 75fcd7db..68c0d2e7 100644 --- a/src/messenger/webim/view/thread_search.php +++ b/src/messenger/webim/view/thread_search.php @@ -22,7 +22,7 @@ $page['menuid'] = "history"; function tpl_content() { global $page, $webimroot; ?> - +