Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MISP / Yeti #1035

Draft
wants to merge 76 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
76 commits
Select commit Hold shift + click to select a range
03c8a83
add invest for one event
sebdraven Mar 1, 2024
5f4b27c
add asn object
sebdraven Mar 1, 2024
f0f4649
ad av signature
sebdraven Mar 1, 2024
b174d14
add btc_wallet
sebdraven Mar 1, 2024
3eaab5c
add c2 list
sebdraven Mar 1, 2024
25d49fe
Update entity.py
sebdraven Mar 5, 2024
98733ea
update location
sebdraven Mar 5, 2024
828d9b6
add setter for location
sebdraven Mar 5, 2024
5986e72
add cmd line and ruff linting
sebdraven Mar 5, 2024
79917b5
add test location
sebdraven Mar 5, 2024
19da92d
add cookie
sebdraven Mar 5, 2024
65370ed
add description
sebdraven Mar 5, 2024
6f25bff
add cookie
sebdraven Mar 5, 2024
f1c39fe
add Jarm
sebdraven Mar 5, 2024
f3ee1d8
add cs-beaconing
sebdraven Mar 5, 2024
c29db8a
Update entity.py
sebdraven Mar 5, 2024
034f006
ruff
sebdraven Mar 5, 2024
f47cecc
ruff
sebdraven Mar 5, 2024
867d254
Update pyproject.toml
sebdraven Mar 5, 2024
a521dc1
ruff linting
sebdraven Mar 5, 2024
8b22aa5
fix test
sebdraven Mar 5, 2024
b898eee
Update entity.py
sebdraven Mar 5, 2024
30f23a8
fix tags
sebdraven Mar 5, 2024
055f1d2
Merge branch 'main' into misp_import
sebdraven Mar 5, 2024
6a7f32e
Update misp_to_yeti.py
sebdraven Mar 5, 2024
a262fd8
update
sebdraven Mar 5, 2024
02071de
Update poetry.lock
sebdraven Mar 5, 2024
731b540
Update misp_to_yeti.py
sebdraven Mar 5, 2024
7ba8343
add test
sebdraven Mar 5, 2024
cc1d285
fixe tags
sebdraven Mar 5, 2024
68f8c53
Update misp_to_yeti.py
sebdraven Mar 5, 2024
013ae02
Update misp_to_yeti.py
sebdraven Mar 5, 2024
ce59669
add pymisp
sebdraven Mar 5, 2024
deb028b
Update misp_to_yeti.py
sebdraven Mar 5, 2024
3ad50ed
Update misp_to_yeti.py
sebdraven Mar 5, 2024
22d6e6d
signature av-test
sebdraven Mar 5, 2024
10edcb9
add command line test
sebdraven Mar 6, 2024
e4f2be4
add cookie object
sebdraven Mar 7, 2024
d90b02a
add crowdsec-ip object
sebdraven Mar 7, 2024
35d4fdd
add cs-beaconing-confi file object
sebdraven Mar 7, 2024
67e16ce
Update misp_event_objects.json
sebdraven Mar 8, 2024
ef3d677
add domain-ip object
sebdraven Mar 8, 2024
8e8e63b
Update misp_event_objects.json
sebdraven Mar 8, 2024
a0c714e
add dns-record object
sebdraven Mar 8, 2024
ad59591
add attribute path
sebdraven Mar 8, 2024
b65e5a6
fix pydantic
sebdraven Mar 8, 2024
6501479
Update observable.py
sebdraven Mar 8, 2024
35d994e
Update observable.py
sebdraven Mar 8, 2024
45ae8c8
Update misp_event_objects.json
sebdraven Mar 11, 2024
9067348
add directory
sebdraven Mar 11, 2024
0278c4d
ruff
sebdraven Mar 11, 2024
1d7639e
add email
sebdraven Mar 11, 2024
2d8d98d
Merge branch 'main' into misp_import
sebdraven Mar 18, 2024
f4660b6
Update misp_event_objects.json
sebdraven Mar 18, 2024
8241b90
add email import and fixes errors import
sebdraven Mar 18, 2024
cde97df
fix error name
sebdraven Mar 18, 2024
f41b680
update emails objects
sebdraven Mar 18, 2024
cd3d5b7
fix linting
sebdraven Mar 18, 2024
7bff7a7
fix variable never used
sebdraven Mar 18, 2024
59aaafe
Merge branch 'main' into misp_import
sebdraven Mar 18, 2024
4ab560c
add exploit entity
sebdraven Mar 18, 2024
5d4a502
Merge branch 'misp_import' of https://github.com/yeti-platform/yeti i…
sebdraven Mar 18, 2024
3305fb7
change attr in exploit
sebdraven Mar 18, 2024
9c28a69
fix tests
sebdraven Mar 18, 2024
4f5f11d
add exploit poc
sebdraven Mar 18, 2024
11ffefb
Update misp_event_objects.json
sebdraven Apr 3, 2024
cf7e56a
Merge branch 'main' into misp_import
sebdraven Apr 3, 2024
d8a9b7f
Update poetry.lock
sebdraven Apr 3, 2024
62c9dbd
Merge branch 'misp_import' of https://github.com/yeti-platform/yeti i…
sebdraven Apr 3, 2024
2ae5d3d
Update poetry.lock
sebdraven Apr 3, 2024
ee57b20
Update pyproject.toml
sebdraven Apr 3, 2024
762621a
Update misp_to_yeti.py
sebdraven Apr 3, 2024
c9c8887
add dateparser
sebdraven Apr 3, 2024
bb178a2
add pycountry
sebdraven Apr 3, 2024
0b6b0fb
Update misp_event_objects.json
sebdraven Apr 3, 2024
3169dff
Merge branch 'main' into misp_import
sebdraven Apr 3, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
991 changes: 961 additions & 30 deletions core/common/misp_to_yeti.py

Large diffs are not rendered by default.

39 changes: 39 additions & 0 deletions core/schemas/entity.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ class EntityType(str, Enum):
tool = "tool"
vulnerability = "vulnerability"
course_of_action = "course-of-action"
location = "location"
exploit = "exploit"


class Entity(YetiTagModel, database_arango.ArangoYetiConnector):
Expand Down Expand Up @@ -137,6 +139,29 @@ class Investigation(Entity):
reference: str = ""


class Location(Entity):
_type_filter: ClassVar[str] = EntityType.location
type: Literal[EntityType.location] = EntityType.location

location: str = ""
reference: str = ""
lat: float = 0.0
lon: float = 0.0
country: str = ""
city: str = ""
country_code: int = 0

def set_country_name_by_code(self, code: int):
import pycountry

self.country = pycountry.countries.get(numeric=str(code)).name

def set_country_code_by_name(self, name: str):
import pycountry

self.country_code = int(pycountry.countries.get(name=name).numeric)


class SeverityType(str, Enum):
none = "none"
low = "low"
Expand Down Expand Up @@ -166,6 +191,17 @@ class Vulnerability(Entity):
reference: str = ""


class Exploit(Entity):
_type_filter: ClassVar[str] = EntityType.exploit
type: Literal[EntityType.exploit] = EntityType.exploit

reference: str = ""
description: str = ""
level: str = ""
software: str = ""
accessibility: str = ""


class CourseOfAction(Entity):
_type_filter: ClassVar[str] = EntityType.course_of_action
type: Literal[EntityType.course_of_action] = EntityType.course_of_action
Expand All @@ -187,6 +223,7 @@ class CourseOfAction(Entity):
EntityType.threat_actor: ThreatActor,
EntityType.tool: Tool,
EntityType.vulnerability: Vulnerability,
EntityType.exploit: Exploit,
}

TYPE_VALIDATOR_MAP = {}
Expand Down Expand Up @@ -225,6 +262,7 @@ def validate_entity(ent: Entity) -> bool:
| ThreatActor
| Tool
| Vulnerability
| Exploit
)


Expand All @@ -242,4 +280,5 @@ def validate_entity(ent: Entity) -> bool:
| Type[ThreatActor]
| Type[Tool]
| Type[Vulnerability]
| Type[Exploit]
)
11 changes: 11 additions & 0 deletions core/schemas/indicator.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class IndicatorType(str, Enum):
sigma = "sigma"
query = "query"
forensicartifact = "forensicartifact"
av_signature = "av_signature"


class IndicatorMatch(BaseModel):
Expand Down Expand Up @@ -314,6 +315,15 @@ def save_indicators(self, create_links: bool = False):
return indicators


class av_signature(Indicator):
_type_filter: ClassVar[str] = IndicatorType.av_signature
type: Literal[IndicatorType.av_signature] = IndicatorType.av_signature
software: str = ""

def match(self, value: str) -> IndicatorMatch | None:
raise NotImplementedError


ARTIFACT_INTERPOLATION_RE = re.compile(r"%%[a-z._]+%%")
ARTIFACT_INTERPOLATION_RE_HKEY_USERS = re.compile(r"HKEY_USERS\\%%users.sid%%")

Expand All @@ -323,6 +333,7 @@ def save_indicators(self, create_links: bool = False):
"sigma": Sigma,
"query": Query,
"forensicartifact": ForensicArtifact,
"av_signature": av_signature,
"indicator": Indicator,
"indicators": Indicator,
}
Expand Down
4 changes: 4 additions & 0 deletions core/schemas/observable.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class ObservableType(str, Enum):
certificate = "certificate"
cidr = "cidr"
command_line = "command_line"
cookie = "cookie"
docker_image = "docker_image"
email = "email"
file = "file"
Expand All @@ -29,6 +30,7 @@ class ObservableType(str, Enum):
imphash = "imphash"
ipv4 = "ipv4"
ipv6 = "ipv6"
jarm = "jarm"
mac_address = "mac_address"
md5 = "md5"
generic = "generic"
Expand Down Expand Up @@ -194,6 +196,7 @@ def find_type(value: str) -> ObservableType | None:
certificate, # noqa: F401
cidr, # noqa: F401
command_line, # noqa: E402, F401
cookie, # noqa: F401
docker_image, # noqa: F401
email, # noqa: F401
file, # noqa: F401
Expand All @@ -203,6 +206,7 @@ def find_type(value: str) -> ObservableType | None:
imphash, # noqa: F401
ipv4, # noqa: F401
ipv6, # noqa: F401
jarm, # noqa: F401
mac_address, # noqa: F401
md5, # noqa: F401
path, # noqa: F401
Expand Down
1 change: 1 addition & 0 deletions core/schemas/observables/asn.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ class ASN(observable.Observable):
type: Literal[observable.ObservableType.asn] = observable.ObservableType.asn
country: str | None = None
description: str | None = None
name: str | None = None


observable.TYPE_MAPPING[observable.ObservableType.asn] = ASN
26 changes: 26 additions & 0 deletions core/schemas/observables/cookie.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import datetime
from typing import Literal, Optional

from core.schemas import observable


class Cookie(observable.Observable):
type: Literal[observable.ObservableType.cookie] = observable.ObservableType.cookie

http_only: bool = False
secure: bool = False
type_cookie: Literal[
"Session management",
"Tracking",
"Personalization",
"Security",
"Exfiltration",
"Beaconing",
"Other",
] = "Session management"
expires: Optional[datetime.datetime] = None
name: Optional[str] = None
cookie: Optional[str] = None


observable.TYPE_MAPPING[observable.ObservableType.cookie] = Cookie
15 changes: 15 additions & 0 deletions core/schemas/observables/jarm.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
from typing import Literal

from core.schemas import observable


class Jarm(observable.Observable):
"""Represents a JARM fingerprint.

Value should be in the form JARM:<HASH>.
"""

type: Literal[observable.ObservableType.jarm] = observable.ObservableType.jarm


observable.TYPE_MAPPING[observable.ObservableType.jarm] = Jarm
5 changes: 5 additions & 0 deletions core/schemas/observables/path.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
from datetime import datetime
from typing import Literal

from core.schemas import observable


class Path(observable.Observable):
type: Literal[observable.ObservableType.path] = observable.ObservableType.path
creation_time: datetime | None = None
modification_time: datetime | None = None
access_time: datetime | None = None
path_encoding: str | None = None


observable.TYPE_MAPPING[observable.ObservableType.path] = Path
12 changes: 8 additions & 4 deletions core/web/apiv2/import_data.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
import json

from fastapi import APIRouter, File, UploadFile

from core.common.misp_to_yeti import MispToYeti

router = APIRouter()


@router.post("/import_misp_json", tags=["import_misp_json"])
async def import_misp_json(misp_file_json: UploadFile = File(...)):
# contents = await misp_file_json.read()
# data_json = json.loads(contents)
contents = await misp_file_json.read()
data_json = json.loads(contents)

# converter = MispToYeti(data_json["Event"])
# converter.misp_to_yeti()
converter = MispToYeti(data_json["Event"])
converter.misp_to_yeti()
return {"status": True}
Loading
Loading