Notes

foreach($companies as $company_id=>$company_name)
{
# skip if file exists and override disabled
if(file_exists("./data/pages/$year/$month/$company_id.html") && $_POST['override-old-pages']!=='1')
{
echo '<br>'. "<span style='color:red'>Skipping $company_id@@$company_name because already exists ...</span>";
continue;
}


$company = $company_id.'@@'.$company_name;

$curl = urlencode("curl 'http://www.sebi.gov.in/sebiweb/other/OtherAction.do?doPmr=yes' -H 'Cookie: JSESSIONID=7BFE72A01EBB4190DA8BE017D4975D93' -H 'Origin: http://www.sebi.gov.in' -H 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36' -H 'Content-Type: application/x-www-form-urlencoded' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0' -H 'Referer: http://www.sebi.gov.in/sebiweb/other/OtherAction.do?doPmr=yes' -H 'Connection: keep-alive' --data 'currdate=&subject_1=-1&name_1=&email_1=&query_1=&capFeedback=&loginflag=0&searchValue=&pmrId=$company&year=$year&month=$month&loginEmail=&loginPassword=&ckvalue=1&moduleNo=-1&moduleId=&link=&yourName=&yourEmail=&friendName=&friendEmail=&mailmessage=&cap_email=' --compressed");

$html = file_get_contents("https://wrael.com/api/curl/v1.php?curl=$curl");

if($html==FALSE)
{echo '<br>'. "<span style='color:red'>Could NOT save $company_id@@$company_name html file ...</span>";
continue;}

if($_POST['test-page-fetch']=='1')
{
echo $html;
exit;
}
else
{

if (!is_dir("./data/pages/$year/$month"))
{
mkdir("./data/pages/$year/$month",0777,TRUE);
}

if(file_put_contents("./data/pages/$year/$month/$company_id.html",$html)!==FALSE)
{
echo '<br>'. "Saved $company_id@@$company_name html file ...";
flush();
}
else
{
echo '<br>'. "<span style='color:red'>Could NOT save $company_id@@$company_name html file ...</span>";
flush();
}
}

}
exit;
}

elseif($_POST['parse-pages']=='1')
{

$year = intval($_POST['year']);
$month = intval($_POST['month']);
if($month<10){$yearmonth = $year.'0'.$month;}else{$yearmonth = $year.$month;}$yearmonth = intval($yearmonth);


$files = scandir("./data/pages/$year/$month");
echo "<br><pre>Files in ./data/pages/$year/$month folder:<br>";print_r($files);echo '<br></pre>';

// loop through each file and add the data to the existing data if not duplicate
foreach($files as $file)
{
// Skip if not html
if(strtolower(pathinfo($file,PATHINFO_EXTENSION)) != 'html')
{continue;}

// fetch file
$html = file_get_contents("./data/pages/$year/$month/$file");


// set company id and name
$data = array();
$parts = explode('.',$file);
$data['company_id'] = $parts[0];
$company_master = json_decode(file_get_contents('companies.json'),TRUE);
$data['company_name'] = $company_master[$data['company_id']];
$data['year'] = $year;
$data['month'] = $month;
$data['yearmonth'] = $yearmonth;


// a new dom object
$dom = new domDocument('1.0', 'utf-8');
// load the html into the object ***/
$dom->loadHTML($html);
//discard white space
$dom->preserveWhiteSpace = false;

// Select the commentary Div
$finder = new DomXPath($dom);

$level_0 = $finder->query('//table');

if($level_0->length == 0)
{
echo '<br>'. "<span style='background-color:red;color:white;'>could not find requested dom node for $file : $data[company_name]</span>";
continue;
}

//echo '<br><pre>';print_r($level_0);echo '<br></pre>';

# initialize variables
$data['num_individual_investors'] = 0;
$data['total_investors'] = 0;
$data['equity_listed_aum'] = 0;
$data['equity_unlisted_aum'] = 0;
$data['debt_plain_aum'] = 0;
$data['debt_structured_aum'] = 0;
$data['equity_derivative_aum'] = 0;
$data['mutual_fund_aum'] = 0;
$data['others_aum'] = 0;
$data['total_aum'] = 0;
$data['gross_sales']=0;
$data['gross_purchases']=0;
$data['turnover_ratio']=0;
$data['performance_pct']=0;
$data['pf_epfo_assets']=0;
$data['advisory_clients_individual']=0;
$data['advisory_clients_corporate']=0;
$data['advisory_aum']=0;
$data['complaints_received']=0;
$data['complaints_resolved']=0;
$data['complaints_at_end']=0;




# parse tables
for($table_number=0;$table_number<$level_0->length;$table_number++)
{
$item = $level_0->item($table_number);

if($item->tagName=='table')
{
$tr = $item->getElementsByTagName('tr');

for($i=0;$i<$tr->length;$i++)
{
$td = $tr->item($i)->getElementsByTagName('td');
for($j=0;$j<$td->length;$j++)
{

// discretionary services table
if($table_number==0)
{
// individual investor count
if($i==3 && $j==2)
{$data['num_individual_investors'] += trim($td->item($j)->nodeValue);}
elseif($i==4 && $j==2)
{$data['num_individual_investors'] += trim($td->item($j)->nodeValue);}


// total investor count
if($i>=3 && $j==2)
{$data['total_investors'] += trim($td->item($j)->nodeValue);}

// various aum
if($i>=3 && $j==3)
{$data['equity_listed_aum'] += trim($td->item($j)->nodeValue);}
elseif($i>=3 && $j==4)
{$data['equity_unlisted_aum'] += trim($td->item($j)->nodeValue);}
elseif($i>=3 && $j==5)
{$data['debt_plain_aum'] += trim($td->item($j)->nodeValue);}
elseif($i>=3 && $j==6)
{$data['debt_structured_aum'] += trim($td->item($j)->nodeValue);}
elseif($i>=3 && $j==7)
{$data['equity_derivative_aum'] += trim($td->item($j)->nodeValue);}
elseif($i>=3 && $j==8)
{$data['mutual_fund_aum'] += trim($td->item($j)->nodeValue);}
elseif($i>=3 && $j==9)
{$data['others_aum'] += trim($td->item($j)->nodeValue);}
elseif($i>=3 && $j==10)
{$data['total_aum'] += trim($td->item($j)->nodeValue);}
}

// discretionary srvices performance table
elseif($table_number==1)
{
if($i==1)
{
if($j==0)
{$data['gross_sales'] += trim($td->item($j)->nodeValue);}
elseif($j==1)
{$data['gross_purchases'] += trim($td->item($j)->nodeValue);}
elseif($j==2)
{$data['turnover_ratio'] += trim($td->item($j)->nodeValue);}
elseif($j==3)
{$data['performance_pct'] += trim($td->item($j)->nodeValue);}
elseif($j==4)
{$data['pf_epfo_assets'] += trim($td->item($j)->nodeValue);}
}
}


// advisory services table
elseif($table_number==4)
{
if($i==1 && $j==2)
{$data['advisory_clients_individual'] += trim($td->item($j)->nodeValue);}
elseif($i==2 && $j==2)
{$data['advisory_clients_corporate'] += trim($td->item($j)->nodeValue);}
elseif($i==3 && $j==2)
{$data['advisory_aum'] += trim($td->item($j)->nodeValue);}
}


// complaints table
elseif($table_number==5)
{
if($i>=1 && $j==3)
{$data['complaints_received'] += trim($td->item($j)->nodeValue);}
elseif($i>=1 && $j==4)
{$data['complaints_resolved'] += trim($td->item($j)->nodeValue);}
elseif($i>=1 && $j==5)
{$data['complaints_at_end'] += trim($td->item($j)->nodeValue);}
}
}
}
}


}