Skip to content

Commit

Permalink
[DB] Turn MAC/OUI details fetch into 100% SQL rather than PHP loops
Browse files Browse the repository at this point in the history 
These loops take a (very) long time and hammer the database. We can do all this in SQL.
  • Loading branch information
@barryo
barryo committed Oct 17, 2017
1 parent 8687f5d commit 333c52b
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 77 deletions.
24 changes: 5 additions & 19 deletions app/Http/Controllers/MacAddressController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,31 +83,17 @@ public function feInit(){
'listColumns' => [
'id' => [ 'title' => 'DB ID', 'display' => false ],

'customer' => [
'title' => 'Customer',
'type' => self::$FE_COL_TYPES[ 'HAS_ONE' ],
'controller' => 'customer',
'action' => 'view',
'idField' => 'customerid'
],

'interface' => [
'title' => 'Interface',
'type' => self::$FE_COL_TYPES[ 'HAS_ONE' ],
'controller' => 'virtual-interface',
'action' => 'edit',
'idField' => 'interfaceid'
],

'ipv4' => 'IPv4',
'ipv6' => 'IPv6',
'customer' => 'Customer',

'ip4' => 'IPv4',
'ip6' => 'IPv6',
'mac' => [
'title' => 'MAC Address',
'type' => self::$FE_COL_TYPES[ 'SCRIPT' ],
'script' => 'mac-address/list-mac-format.foil.php'
],

'manufacturer' => 'Manufacturer'
'organisation' => 'Manufacturer'
],
];

Expand Down
5 changes: 4 additions & 1 deletion config/doctrine.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,10 @@
| DQL custom string functions
|--------------------------------------------------------------------------
*/
'custom_string_functions' => [],
'custom_string_functions' => [
'GROUP_CONCAT' => DoctrineExtensions\Query\Mysql\GroupConcat::class,
],

/*
|--------------------------------------------------------------------------
| Enable query logging with laravel file logging,
Expand Down
97 changes: 40 additions & 57 deletions database/Repositories/MACAddress.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,82 +39,65 @@ public function findVirtualInterface( $mac )
}

/**
* Get all infrastructures (or a particular one) for listing on the frontend CRUD
* Get all MAC addresses with interface and OUI organisation details (or a particular one) for listing on the frontend CRUD
*
* @see \IXP\Http\Controller\Doctrine2Frontend
* Returns an array of the form:
*
* array:12 [
* "id" => 4848209
* "firstseen" => DateTime object
* "lastseen" => null
* "mac" => "00d0ffaaaaaa"
* "customerid" => 101
* "customer" => "Someone"
* "viid" => 203
* "switchname" => "swi2-deg1-4"
* "switchport" => "GigabitEthernet1/1,GigabitEthernet1/2" // ** NB: multiple's separated by comma
* "ip4" => "194.88.240.33" // ** NB: multiple's separated by comma
* "ip6" => "2001:7f8:18:12::33" // ** NB: multiple's separated by comma
* "organisation" => "Cisco Systems, Inc"
* ]
*
* @see \IXP\Http\Controllers\Doctrine2Frontend
*
* @param \stdClass $feParams
* @param int|null $id
* @return array Array of infrastructures (as associated arrays) (or single element if `$id` passed)
* @return array Array of MAC address entries (as associated arrays) (or single element if `$id` passed)
*/
public function getAllForFeList( \stdClass $feParams, int $id = null )
{
$dql = "SELECT m
$dql = "SELECT m.id AS id, m.firstseen AS firstseen, m.lastseen AS lastseen, m.mac AS mac,
c.id AS customerid, c.abbreviatedName AS customer,
vi.id AS viid, s.name AS switchname,
GROUP_CONCAT( sp.name ) AS switchport,
GROUP_CONCAT( DISTINCT ipv4.address ) AS ip4,
GROUP_CONCAT( DISTINCT ipv6.address ) AS ip6,
COALESCE( o.organisation, 'Unknown' ) AS organisation
FROM Entities\\MACAddress m
JOIN m.VirtualInterface vi
JOIN vi.VlanInterfaces vli
JOIN vi.Customer c
JOIN vi.PhysicalInterfaces pi
JOIN pi.SwitchPort sp
JOIN sp.Switcher s
JOIN m.VirtualInterface vi
JOIN vi.VlanInterfaces vli
JOIN vli.IPv4Address ipv4
JOIN vli.IPv6Address ipv6
JOIN vi.Customer c
JOIN vi.PhysicalInterfaces pi
JOIN pi.SwitchPort sp
JOIN sp.Switcher s
LEFT JOIN Entities\\OUI o WITH SUBSTRING( m.mac, 1, 6 ) = o.oui
WHERE 1 = 1";

if( $id ) {
$dql .= " AND m.id = " . (int)$id;
}

$dql .= " GROUP BY m.id, m.firstseen, m.lastseen, m.mac, c.id, c.abbreviatedName, vi.id, s.name, o.organisation";

if( isset( $feParams->listOrderBy ) ) {
$dql .= " ORDER BY c.name ";
$dql .= isset( $feParams->listOrderByDir ) ? $feParams->listOrderByDir : 'ASC';
}

$query = $this->getEntityManager()->createQuery( $dql );

$objects = $query->getResult();

$data = [];
$datas = [];

foreach( $objects as $m ) {
$data[ $m->getId() ]['id'] = $m->getId();
$data[ $m->getId() ]['firstseen'] = $m->getFirstseen();
$data[ $m->getId() ]['lastseen'] = $m->getLastseen();
$data[ $m->getId() ]['mac'] = $m->getMac();
$data[ $m->getId() ]['customerid'] = $m->getVirtualInterface()->getCustomer()->getId();
$data[ $m->getId() ]['customer'] = $m->getVirtualInterface()->getCustomer()->getName();
$data[ $m->getId() ]['interfaceid'] = $m->getVirtualInterface()->getId();

$data[ $m->getId() ]['interface'] =
$m->getVirtualInterface()->getPhysicalInterfaces()[0]->getSwitchport()->getSwitcher()->getName() . " - "
. $m->getVirtualInterface()->getPhysicalInterfaces()[0]->getSwitchport()->getName();

if( $m->getVirtualInterface()->getVlanInterfaces() ) {
foreach( $m->getVirtualInterface()->getVlanInterfaces() as $vli ) {
if( !$vli->getVlan()->getPrivate() ) {
if( $vli->getIpv4Address() )
$data[ $m->getId() ]['ipv4'] = $vli->getIpv4Address()->getAddress();

if( $vli->getIpv6Address() )
$data[ $m->getId() ]['ipv6'] = $vli->getIpv6Address()->getAddress();

break;
}
}

if( !isset( $data[ $m->getId() ]['ipv4'] ) )
$data[ $m->getId() ]['ipv4'] = '';

if( !isset( $data[ $m->getId() ]['ipv6'] ) )
$data[ $m->getId() ]['ipv6'] = '';
}

if( $m->getMac() ){
$data[ $m->getId() ]['manufacturer'] = D2EM::getRepository( OUIEntity::class )->getOrganisation( strtolower( substr( $m->getMac(), 0, 6 ) ) );
}

}

return $data;
return $this->getEntityManager()->createQuery( $dql )->getArrayResult();
}
}

0 comments on commit 333c52b

Please sign in to comment.