<?php

[0] Fix | Delete

[1] Fix | Delete

/**

[2] Fix | Delete

* The Crawler Sitemap Class

[3] Fix | Delete

*

[4] Fix | Delete

* @since 1.1.0

[5] Fix | Delete

*/

[6] Fix | Delete

[7] Fix | Delete

namespace LiteSpeed;

[8] Fix | Delete

[9] Fix | Delete

defined('WPINC') || exit();

[10] Fix | Delete

[11] Fix | Delete

class Crawler_Map extends Root

[12] Fix | Delete

{

[13] Fix | Delete

const LOG_TAG = '🐞🗺️';

[14] Fix | Delete

[15] Fix | Delete

const BM_MISS = 1;

[16] Fix | Delete

const BM_HIT = 2;

[17] Fix | Delete

const BM_BLACKLIST = 4;

[18] Fix | Delete

[19] Fix | Delete

private $_home_url; // Used to simplify urls

[20] Fix | Delete

private $_tb;

[21] Fix | Delete

private $_tb_blacklist;

[22] Fix | Delete

private $__data;

[23] Fix | Delete

private $_conf_map_timeout;

[24] Fix | Delete

private $_urls = array();

[25] Fix | Delete

[26] Fix | Delete

/**

[27] Fix | Delete

* Instantiate the class

[28] Fix | Delete

*

[29] Fix | Delete

* @since 1.1.0

[30] Fix | Delete

*/

[31] Fix | Delete

public function __construct()

[32] Fix | Delete

{

[33] Fix | Delete

$this->_home_url = get_home_url();

[34] Fix | Delete

$this->__data = Data::cls();

[35] Fix | Delete

$this->_tb = $this->__data->tb('crawler');

[36] Fix | Delete

$this->_tb_blacklist = $this->__data->tb('crawler_blacklist');

[37] Fix | Delete

$this->_conf_map_timeout = $this->conf(Base::O_CRAWLER_MAP_TIMEOUT);

[38] Fix | Delete

}

[39] Fix | Delete

[40] Fix | Delete

/**

[41] Fix | Delete

* Save URLs crawl status into DB

[42] Fix | Delete

*

[43] Fix | Delete

* @since 3.0

[44] Fix | Delete

* @access public

[45] Fix | Delete

*/

[46] Fix | Delete

public function save_map_status($list, $curr_crawler)

[47] Fix | Delete

{

[48] Fix | Delete

global $wpdb;

[49] Fix | Delete

Utility::compatibility();

[50] Fix | Delete

[51] Fix | Delete

$total_crawler = count(Crawler::cls()->list_crawlers());

[52] Fix | Delete

$total_crawler_pos = $total_crawler - 1;

[53] Fix | Delete

[54] Fix | Delete

// Replace current crawler's position

[55] Fix | Delete

$curr_crawler = (int) $curr_crawler;

[56] Fix | Delete

foreach ($list as $bit => $ids) {

[57] Fix | Delete

// $ids = [ id => [ url, code ], ... ]

[58] Fix | Delete

if (!$ids) {

[59] Fix | Delete

continue;

[60] Fix | Delete

}

[61] Fix | Delete

self::debug("Update map [crawler] $curr_crawler [bit] $bit [count] " . count($ids));

[62] Fix | Delete

[63] Fix | Delete

// Update res first, then reason

[64] Fix | Delete

$right_pos = $total_crawler_pos - $curr_crawler;

[65] Fix | Delete

$sql_res = "CONCAT( LEFT( res, $curr_crawler ), '$bit', RIGHT( res, $right_pos ) )";

[66] Fix | Delete

[67] Fix | Delete

$id_all = implode(',', array_map('intval', array_keys($ids)));

[68] Fix | Delete

[69] Fix | Delete

$wpdb->query("UPDATE `$this->_tb` SET res = $sql_res WHERE id IN ( $id_all )");

[70] Fix | Delete

[71] Fix | Delete

// Add blacklist

[72] Fix | Delete

if ($bit == 'B' || $bit == 'N') {

[73] Fix | Delete

$q = "SELECT a.id, a.url FROM `$this->_tb_blacklist` a LEFT JOIN `$this->_tb` b ON b.url=a.url WHERE b.id IN ( $id_all )";

[74] Fix | Delete

$existing = $wpdb->get_results($q, ARRAY_A);

[75] Fix | Delete

// Update current crawler status tag in existing blacklist

[76] Fix | Delete

if ($existing) {

[77] Fix | Delete

$count = $wpdb->query("UPDATE `$this->_tb_blacklist` SET res = $sql_res WHERE id IN ( " . implode(',', array_column($existing, 'id')) . ' )');

[78] Fix | Delete

self::debug('Update blacklist [count] ' . $count);

[79] Fix | Delete

}

[80] Fix | Delete

[81] Fix | Delete

// Append new blacklist

[82] Fix | Delete

if (count($ids) > count($existing)) {

[83] Fix | Delete

$new_urls = array_diff(array_column($ids, 'url'), array_column($existing, 'url'));

[84] Fix | Delete

[85] Fix | Delete

self::debug('Insert into blacklist [count] ' . count($new_urls));

[86] Fix | Delete

[87] Fix | Delete

$q = "INSERT INTO `$this->_tb_blacklist` ( url, res, reason ) VALUES " . implode(',', array_fill(0, count($new_urls), '( %s, %s, %s )'));

[88] Fix | Delete

$data = array();

[89] Fix | Delete

$res = array_fill(0, $total_crawler, '-');

[90] Fix | Delete

$res[$curr_crawler] = $bit;

[91] Fix | Delete

$res = implode('', $res);

[92] Fix | Delete

$default_reason = $total_crawler > 1 ? str_repeat(',', $total_crawler - 1) : ''; // Pre-populate default reason value first, update later

[93] Fix | Delete

foreach ($new_urls as $url) {

[94] Fix | Delete

$data[] = $url;

[95] Fix | Delete

$data[] = $res;

[96] Fix | Delete

$data[] = $default_reason;

[97] Fix | Delete

}

[98] Fix | Delete

$wpdb->query($wpdb->prepare($q, $data));

[99] Fix | Delete

}

[100] Fix | Delete

}

[101] Fix | Delete

[102] Fix | Delete

// Update sitemap reason w/ HTTP code

[103] Fix | Delete

$reason_array = array();

[104] Fix | Delete

foreach ($ids as $id => $v2) {

[105] Fix | Delete

$code = (int) $v2['code'];

[106] Fix | Delete

if (empty($reason_array[$code])) {

[107] Fix | Delete

$reason_array[$code] = array();

[108] Fix | Delete

}

[109] Fix | Delete

$reason_array[$code][] = (int) $id;

[110] Fix | Delete

}

[111] Fix | Delete

[112] Fix | Delete

foreach ($reason_array as $code => $v2) {

[113] Fix | Delete

// Complement comma

[114] Fix | Delete

if ($curr_crawler) {

[115] Fix | Delete

$code = ',' . $code;

[116] Fix | Delete

}

[117] Fix | Delete

if ($curr_crawler < $total_crawler_pos) {

[118] Fix | Delete

$code .= ',';

[119] Fix | Delete

}

[120] Fix | Delete

[121] Fix | Delete

$count = $wpdb->query(

[122] Fix | Delete

"UPDATE `$this->_tb` SET reason=CONCAT(SUBSTRING_INDEX(reason, ',', $curr_crawler), '$code', SUBSTRING_INDEX(reason, ',', -$right_pos)) WHERE id IN (" .

[123] Fix | Delete

implode(',', $v2) .

[124] Fix | Delete

')'

[125] Fix | Delete

);

[126] Fix | Delete

[127] Fix | Delete

self::debug("Update map reason [code] $code [pos] left $curr_crawler right -$right_pos [count] $count");

[128] Fix | Delete

[129] Fix | Delete

// Update blacklist reason

[130] Fix | Delete

if ($bit == 'B' || $bit == 'N') {

[131] Fix | Delete

$count = $wpdb->query(

[132] Fix | Delete

"UPDATE `$this->_tb_blacklist` a LEFT JOIN `$this->_tb` b ON b.url = a.url SET a.reason=CONCAT(SUBSTRING_INDEX(a.reason, ',', $curr_crawler), '$code', SUBSTRING_INDEX(a.reason, ',', -$right_pos)) WHERE b.id IN (" .

[133] Fix | Delete

implode(',', $v2) .

[134] Fix | Delete

')'

[135] Fix | Delete

);

[136] Fix | Delete

[137] Fix | Delete

self::debug("Update blacklist [code] $code [pos] left $curr_crawler right -$right_pos [count] $count");

[138] Fix | Delete

}

[139] Fix | Delete

}

[140] Fix | Delete

// Reset list

[141] Fix | Delete

$list[$bit] = array();

[142] Fix | Delete

}

[143] Fix | Delete

[144] Fix | Delete

return $list;

[145] Fix | Delete

}

[146] Fix | Delete

[147] Fix | Delete

/**

[148] Fix | Delete

* Add one record to blacklist

[149] Fix | Delete

* NOTE: $id is sitemap table ID

[150] Fix | Delete

*

[151] Fix | Delete

* @since 3.0

[152] Fix | Delete

* @access public

[153] Fix | Delete

*/

[154] Fix | Delete

public function blacklist_add($id)

[155] Fix | Delete

{

[156] Fix | Delete

global $wpdb;

[157] Fix | Delete

[158] Fix | Delete

$id = (int) $id;

[159] Fix | Delete

[160] Fix | Delete

// Build res&reason

[161] Fix | Delete

$total_crawler = count(Crawler::cls()->list_crawlers());

[162] Fix | Delete

$res = str_repeat('B', $total_crawler);

[163] Fix | Delete

$reason = implode(',', array_fill(0, $total_crawler, 'Man'));

[164] Fix | Delete

[165] Fix | Delete

$row = $wpdb->get_row("SELECT a.url, b.id FROM `$this->_tb` a LEFT JOIN `$this->_tb_blacklist` b ON b.url = a.url WHERE a.id = '$id'", ARRAY_A);

[166] Fix | Delete

if (!$row) {

[167] Fix | Delete

self::debug('blacklist failed to add [id] ' . $id);

[168] Fix | Delete

return;

[169] Fix | Delete

}

[170] Fix | Delete

[171] Fix | Delete

self::debug('Add to blacklist [url] ' . $row['url']);

[172] Fix | Delete

[173] Fix | Delete

$q = "UPDATE `$this->_tb` SET res = %s, reason = %s WHERE id = %d";

[174] Fix | Delete

$wpdb->query($wpdb->prepare($q, array($res, $reason, $id)));

[175] Fix | Delete

[176] Fix | Delete

if ($row['id']) {

[177] Fix | Delete

$q = "UPDATE `$this->_tb_blacklist` SET res = %s, reason = %s WHERE id = %d";

[178] Fix | Delete

$wpdb->query($wpdb->prepare($q, array($res, $reason, $row['id'])));

[179] Fix | Delete

} else {

[180] Fix | Delete

$q = "INSERT INTO `$this->_tb_blacklist` (url, res, reason) VALUES (%s, %s, %s)";

[181] Fix | Delete

$wpdb->query($wpdb->prepare($q, array($row['url'], $res, $reason)));

[182] Fix | Delete

}

[183] Fix | Delete

}

[184] Fix | Delete

[185] Fix | Delete

/**

[186] Fix | Delete

* Delete one record from blacklist

[187] Fix | Delete

*

[188] Fix | Delete

* @since 3.0

[189] Fix | Delete

* @access public

[190] Fix | Delete

*/

[191] Fix | Delete

public function blacklist_del($id)

[192] Fix | Delete

{

[193] Fix | Delete

global $wpdb;

[194] Fix | Delete

if (!$this->__data->tb_exist('crawler_blacklist')) {

[195] Fix | Delete

return;

[196] Fix | Delete

}

[197] Fix | Delete

[198] Fix | Delete

$id = (int) $id;

[199] Fix | Delete

self::debug('blacklist delete [id] ' . $id);

[200] Fix | Delete

[201] Fix | Delete

$wpdb->query("UPDATE `$this->_tb` SET res=REPLACE(REPLACE(res, 'N', '-'), 'B', '-') WHERE url=(SELECT url FROM `$this->_tb_blacklist` WHERE id='$id')");

[202] Fix | Delete

$wpdb->query("DELETE FROM `$this->_tb_blacklist` WHERE id='$id'");

[203] Fix | Delete

}

[204] Fix | Delete

[205] Fix | Delete

/**

[206] Fix | Delete

* Empty blacklist

[207] Fix | Delete

*

[208] Fix | Delete

* @since 3.0

[209] Fix | Delete

* @access public

[210] Fix | Delete

*/

[211] Fix | Delete

public function blacklist_empty()

[212] Fix | Delete

{

[213] Fix | Delete

global $wpdb;

[214] Fix | Delete

[215] Fix | Delete

if (!$this->__data->tb_exist('crawler_blacklist')) {

[216] Fix | Delete

return;

[217] Fix | Delete

}

[218] Fix | Delete

[219] Fix | Delete

self::debug('Truncate blacklist');

[220] Fix | Delete

$wpdb->query("UPDATE `$this->_tb` SET res=REPLACE(REPLACE(res, 'N', '-'), 'B', '-')");

[221] Fix | Delete

$wpdb->query("TRUNCATE `$this->_tb_blacklist`");

[222] Fix | Delete

}

[223] Fix | Delete

[224] Fix | Delete

/**

[225] Fix | Delete

* List blacklist

[226] Fix | Delete

*

[227] Fix | Delete

* @since 3.0

[228] Fix | Delete

* @access public

[229] Fix | Delete

*/

[230] Fix | Delete

public function list_blacklist($limit = false, $offset = false)

[231] Fix | Delete

{

[232] Fix | Delete

global $wpdb;

[233] Fix | Delete

[234] Fix | Delete

if (!$this->__data->tb_exist('crawler_blacklist')) {

[235] Fix | Delete

return array();

[236] Fix | Delete

}

[237] Fix | Delete

[238] Fix | Delete

$q = "SELECT * FROM `$this->_tb_blacklist` ORDER BY id DESC";

[239] Fix | Delete

[240] Fix | Delete

if ($limit !== false) {

[241] Fix | Delete

if ($offset === false) {

[242] Fix | Delete

$total = $this->count_blacklist();

[243] Fix | Delete

$offset = Utility::pagination($total, $limit, true);

[244] Fix | Delete

}

[245] Fix | Delete

$q .= ' LIMIT %d, %d';

[246] Fix | Delete

$q = $wpdb->prepare($q, $offset, $limit);

[247] Fix | Delete

}

[248] Fix | Delete

return $wpdb->get_results($q, ARRAY_A);

[249] Fix | Delete

}

[250] Fix | Delete

[251] Fix | Delete

/**

[252] Fix | Delete

* Count blacklist

[253] Fix | Delete

*/

[254] Fix | Delete

public function count_blacklist()

[255] Fix | Delete

{

[256] Fix | Delete

global $wpdb;

[257] Fix | Delete

[258] Fix | Delete

if (!$this->__data->tb_exist('crawler_blacklist')) {

[259] Fix | Delete

return false;

[260] Fix | Delete

}

[261] Fix | Delete

[262] Fix | Delete

$q = "SELECT COUNT(*) FROM `$this->_tb_blacklist`";

[263] Fix | Delete

return $wpdb->get_var($q);

[264] Fix | Delete

}

[265] Fix | Delete

[266] Fix | Delete

/**

[267] Fix | Delete

* Empty sitemap

[268] Fix | Delete

*

[269] Fix | Delete

* @since 3.0

[270] Fix | Delete

* @access public

[271] Fix | Delete

*/

[272] Fix | Delete

public function empty_map()

[273] Fix | Delete

{

[274] Fix | Delete

Data::cls()->tb_del('crawler');

[275] Fix | Delete

[276] Fix | Delete

$msg = __('Sitemap cleaned successfully', 'litespeed-cache');

[277] Fix | Delete

Admin_Display::succeed($msg);

[278] Fix | Delete

}

[279] Fix | Delete

[280] Fix | Delete

/**

[281] Fix | Delete

* List generated sitemap

[282] Fix | Delete

*

[283] Fix | Delete

* @since 3.0

[284] Fix | Delete

* @access public

[285] Fix | Delete

*/

[286] Fix | Delete

public function list_map($limit, $offset = false)

[287] Fix | Delete

{

[288] Fix | Delete

global $wpdb;

[289] Fix | Delete

[290] Fix | Delete

if (!$this->__data->tb_exist('crawler')) {

[291] Fix | Delete

return array();

[292] Fix | Delete

}

[293] Fix | Delete

[294] Fix | Delete

if ($offset === false) {

[295] Fix | Delete

$total = $this->count_map();

[296] Fix | Delete

$offset = Utility::pagination($total, $limit, true);

[297] Fix | Delete

}

[298] Fix | Delete

[299] Fix | Delete

$type = Router::verify_type();

[300] Fix | Delete

[301] Fix | Delete

$where = '';

[302] Fix | Delete

if (!empty($_POST['kw'])) {

[303] Fix | Delete

$q = "SELECT * FROM `$this->_tb` WHERE url LIKE %s";

[304] Fix | Delete

if ($type == 'hit') {

[305] Fix | Delete

$q .= " AND res LIKE '%H%'";

[306] Fix | Delete

}

[307] Fix | Delete

if ($type == 'miss') {

[308] Fix | Delete

$q .= " AND res LIKE '%M%'";

[309] Fix | Delete

}

[310] Fix | Delete

if ($type == 'blacklisted') {

[311] Fix | Delete

$q .= " AND res LIKE '%B%'";

[312] Fix | Delete

}

[313] Fix | Delete

$q .= ' ORDER BY id LIMIT %d, %d';

[314] Fix | Delete

$where = '%' . $wpdb->esc_like($_POST['kw']) . '%';

[315] Fix | Delete

return $wpdb->get_results($wpdb->prepare($q, $where, $offset, $limit), ARRAY_A);

[316] Fix | Delete

}

[317] Fix | Delete

[318] Fix | Delete

$q = "SELECT * FROM `$this->_tb`";

[319] Fix | Delete

if ($type == 'hit') {

[320] Fix | Delete

$q .= " WHERE res LIKE '%H%'";

[321] Fix | Delete

}

[322] Fix | Delete

if ($type == 'miss') {

[323] Fix | Delete

$q .= " WHERE res LIKE '%M%'";

[324] Fix | Delete

}

[325] Fix | Delete

if ($type == 'blacklisted') {

[326] Fix | Delete

$q .= " WHERE res LIKE '%B%'";

[327] Fix | Delete

}

[328] Fix | Delete

$q .= ' ORDER BY id LIMIT %d, %d';

[329] Fix | Delete

// self::debug("q=$q offset=$offset, limit=$limit");

[330] Fix | Delete

return $wpdb->get_results($wpdb->prepare($q, $offset, $limit), ARRAY_A);

[331] Fix | Delete

}

[332] Fix | Delete

[333] Fix | Delete

/**

[334] Fix | Delete

* Count sitemap

[335] Fix | Delete

*/

[336] Fix | Delete

public function count_map()

[337] Fix | Delete

{

[338] Fix | Delete

global $wpdb;

[339] Fix | Delete

[340] Fix | Delete

if (!$this->__data->tb_exist('crawler')) {

[341] Fix | Delete

return false;

[342] Fix | Delete

}

[343] Fix | Delete

[344] Fix | Delete

$q = "SELECT COUNT(*) FROM `$this->_tb`";

[345] Fix | Delete

[346] Fix | Delete

$type = Router::verify_type();

[347] Fix | Delete

if ($type == 'hit') {

[348] Fix | Delete

$q .= " WHERE res LIKE '%H%'";

[349] Fix | Delete

}

[350] Fix | Delete

if ($type == 'miss') {

[351] Fix | Delete

$q .= " WHERE res LIKE '%M%'";

[352] Fix | Delete

}

[353] Fix | Delete

if ($type == 'blacklisted') {

[354] Fix | Delete

$q .= " WHERE res LIKE '%B%'";

[355] Fix | Delete

}

[356] Fix | Delete

[357] Fix | Delete

return $wpdb->get_var($q);

[358] Fix | Delete

}

[359] Fix | Delete

[360] Fix | Delete

/**

[361] Fix | Delete

* Generate sitemap

[362] Fix | Delete

*

[363] Fix | Delete

* @since 1.1.0

[364] Fix | Delete

* @access public

[365] Fix | Delete

*/

[366] Fix | Delete

public function gen($manual = false)

[367] Fix | Delete

{

[368] Fix | Delete

$count = $this->_gen();

[369] Fix | Delete

[370] Fix | Delete

if (!$count) {

[371] Fix | Delete

Admin_Display::error(__('No valid sitemap parsed for crawler.', 'litespeed-cache'));

[372] Fix | Delete

return;

[373] Fix | Delete

}

[374] Fix | Delete

[375] Fix | Delete

if (!defined('DOING_CRON') && $manual) {

[376] Fix | Delete

$msg = sprintf(__('Sitemap created successfully: %d items', 'litespeed-cache'), $count);

[377] Fix | Delete

Admin_Display::succeed($msg);

[378] Fix | Delete

}

[379] Fix | Delete

}

[380] Fix | Delete

[381] Fix | Delete

/**

[382] Fix | Delete

* Generate the sitemap

[383] Fix | Delete

*

[384] Fix | Delete

* @since 1.1.0

[385] Fix | Delete

* @access private

[386] Fix | Delete

*/

[387] Fix | Delete

private function _gen()

[388] Fix | Delete

{

[389] Fix | Delete

global $wpdb;

[390] Fix | Delete

[391] Fix | Delete

if (!$this->__data->tb_exist('crawler')) {

[392] Fix | Delete

$this->__data->tb_create('crawler');

[393] Fix | Delete

}

[394] Fix | Delete

[395] Fix | Delete

if (!$this->__data->tb_exist('crawler_blacklist')) {

[396] Fix | Delete

$this->__data->tb_create('crawler_blacklist');

[397] Fix | Delete

}

[398] Fix | Delete

[399] Fix | Delete

// use custom sitemap

[400] Fix | Delete

if (!($sitemap = $this->conf(Base::O_CRAWLER_SITEMAP))) {

[401] Fix | Delete

return false;

[402] Fix | Delete

}

[403] Fix | Delete

[404] Fix | Delete

$offset = strlen($this->_home_url);

[405] Fix | Delete

$sitemap = Utility::sanitize_lines($sitemap);

[406] Fix | Delete

[407] Fix | Delete

try {

[408] Fix | Delete

foreach ($sitemap as $this_map) {

[409] Fix | Delete

$this->_parse($this_map);

[410] Fix | Delete

}

[411] Fix | Delete

} catch (\Exception $e) {

[412] Fix | Delete

self::debug('❌ failed to parse custom sitemap: ' . $e->getMessage());

[413] Fix | Delete

}

[414] Fix | Delete

[415] Fix | Delete

if (is_array($this->_urls) && !empty($this->_urls)) {

[416] Fix | Delete

if ($this->conf(Base::O_CRAWLER_DROP_DOMAIN)) {

[417] Fix | Delete

foreach ($this->_urls as $k => $v) {

[418] Fix | Delete

if (stripos($v, $this->_home_url) !== 0) {

[419] Fix | Delete

unset($this->_urls[$k]);

[420] Fix | Delete

continue;

[421] Fix | Delete

}

[422] Fix | Delete

$this->_urls[$k] = substr($v, $offset);

[423] Fix | Delete

}

[424] Fix | Delete

}

[425] Fix | Delete

[426] Fix | Delete

$this->_urls = array_unique($this->_urls);

[427] Fix | Delete

}

[428] Fix | Delete

[429] Fix | Delete

self::debug('Truncate sitemap');

[430] Fix | Delete

$wpdb->query("TRUNCATE `$this->_tb`");

[431] Fix | Delete

[432] Fix | Delete

self::debug('Generate sitemap');

[433] Fix | Delete

[434] Fix | Delete

// Filter URLs in blacklist

[435] Fix | Delete

$blacklist = $this->list_blacklist();

[436] Fix | Delete

[437] Fix | Delete

$full_blacklisted = array();

[438] Fix | Delete

$partial_blacklisted = array();

[439] Fix | Delete

foreach ($blacklist as $v) {

[440] Fix | Delete

if (strpos($v['res'], '-') === false) {

[441] Fix | Delete

// Full blacklisted

[442] Fix | Delete

$full_blacklisted[] = $v['url'];

[443] Fix | Delete

} else {

[444] Fix | Delete

// Replace existing reason

[445] Fix | Delete

$v['reason'] = explode(',', $v['reason']);

[446] Fix | Delete

$v['reason'] = array_map(function ($element) {

[447] Fix | Delete

return $element ? 'Existed' : '';

[448] Fix | Delete

}, $v['reason']);

[449] Fix | Delete

$v['reason'] = implode(',', $v['reason']);

[450] Fix | Delete

$partial_blacklisted[$v['url']] = array(

[451] Fix | Delete

'res' => $v['res'],

[452] Fix | Delete

'reason' => $v['reason'],

[453] Fix | Delete

);

[454] Fix | Delete

}

[455] Fix | Delete

}

[456] Fix | Delete

[457] Fix | Delete

// Drop all blacklisted URLs

[458] Fix | Delete

$this->_urls = array_diff($this->_urls, $full_blacklisted);

[459] Fix | Delete

[460] Fix | Delete

// Default res & reason

[461] Fix | Delete

$crawler_count = count(Crawler::cls()->list_crawlers());

[462] Fix | Delete

$default_res = str_repeat('-', $crawler_count);

[463] Fix | Delete

$default_reason = $crawler_count > 1 ? str_repeat(',', $crawler_count - 1) : '';

[464] Fix | Delete

[465] Fix | Delete

$data = array();

[466] Fix | Delete

foreach ($this->_urls as $url) {

[467] Fix | Delete

$data[] = $url;

[468] Fix | Delete

$data[] = array_key_exists($url, $partial_blacklisted) ? $partial_blacklisted[$url]['res'] : $default_res;

[469] Fix | Delete

$data[] = array_key_exists($url, $partial_blacklisted) ? $partial_blacklisted[$url]['reason'] : $default_reason;

[470] Fix | Delete

}

[471] Fix | Delete

[472] Fix | Delete

foreach (array_chunk($data, 300) as $data2) {

[473] Fix | Delete

$this->_save($data2);

[474] Fix | Delete

}

[475] Fix | Delete

[476] Fix | Delete

// Reset crawler

[477] Fix | Delete

Crawler::cls()->reset_pos();

[478] Fix | Delete

[479] Fix | Delete

return count($this->_urls);

[480] Fix | Delete

}

[481] Fix | Delete

[482] Fix | Delete

/**

[483] Fix | Delete

* Save data to table

[484] Fix | Delete

*

[485] Fix | Delete

* @since 3.0

[486] Fix | Delete

* @access private

[487] Fix | Delete

*/

[488] Fix | Delete

private function _save($data, $fields = 'url,res,reason')

[489] Fix | Delete

{

[490] Fix | Delete

global $wpdb;

[491] Fix | Delete

[492] Fix | Delete

if (empty($data)) {

[493] Fix | Delete

return;

[494] Fix | Delete

}

[495] Fix | Delete

[496] Fix | Delete

$q = "INSERT INTO `$this->_tb` ( $fields ) VALUES ";

[497] Fix | Delete

[498] Fix | Delete

// Add placeholder

[499] Fix | Delete