Category: AWS

A contents of Amazon Web Services

04/18/2024

sam build 시 SSL 접속 에러 해결 방

다음과 같이 sam build 시 SSL 접속 에러가 발생할 수 있다.

]$ sam build --template template.yaml
Starting Build use cache
Manifest file is changed (new hash: 3298f13049d19cffaa37ca931dd4d421) or dependency folder (.aws-sam\deps\53036498-ac29-43da-b665-6ca82b0cd06d) is missing for (HelloWorldFunction), downloading
dependencies and copying/building source
Building codeuri: C:\labs\aws\Source_Code\lambda\make-thumnail\hello_world runtime: python3.12 metadata: {} architecture: x86_64 functions: HelloWorldFunction
 Running PythonPipBuilder:CleanUp
 Running PythonPipBuilder:ResolveDependencies

Build Failed
Error: PythonPipBuilder:ResolveDependencies - WARNING: pip is being invoked by an old script wrapper. This will fail in a future version of pip.
Please see https://github.com/pypa/pip/issues/5599 for advice on fixing the underlying issue.
To avoid this problem you can invoke Python with '-m pip' instead of running pip directly.
  WARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata
  WARNING: Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata
  WARNING: Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata
  WARNING: Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata
  WARNING: Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata
ERROR: Exception:
Traceback (most recent call last):
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 714, in urlopen
    httplib_response = self._make_request(
                       ^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 403, in _make_request
    self._validate_conn(conn)
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 1053, in _validate_conn
    conn.connect()
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connection.py", line 419, in connect
    self.sock = ssl_wrap_socket(
                ^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\util\ssl_.py", line 449, in ssl_wrap_socket
    ssl_sock = _ssl_wrap_socket_impl(
               ^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\util\ssl_.py", line 493, in _ssl_wrap_socket_impl
    return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Python312\Lib\ssl.py", line 455, in wrap_socket
    return self.sslsocket_class._create(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Python312\Lib\ssl.py", line 1046, in _create
    self.do_handshake()
  File "c:\Python312\Lib\ssl.py", line 1321, in do_handshake
    self._sslobj.do_handshake()
ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\adapters.py", line 486, in send
    resp = conn.urlopen(
           ^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 826, in urlopen
    return self.urlopen(
           ^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 826, in urlopen
    return self.urlopen(
           ^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 826, in urlopen
    return self.urlopen(
           ^^^^^^^^^^^^^
  [Previous line repeated 2 more times]
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 798, in urlopen
    retries = retries.increment(
              ^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\util\retry.py", line 592, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
pip._vendor.urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='files.pythonhosted.org', port=443): Max retries exceeded with url: /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)')))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "c:\labs\aws\Lib\site-packages\pip\_internal\cli\base_command.py", line 180, in exc_logging_wrapper
    status = run_func(*args)
             ^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\cli\req_command.py", line 245, in wrapper
    return func(self, options, args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\commands\download.py", line 132, in run
    requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\resolver.py", line 95, in resolve
    result = self._result = resolver.resolve(
                            ^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 546, in resolve
    state = resolution.resolve(requirements, max_rounds=max_rounds)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 397, in resolve
    self._add_to_criteria(self.state.criteria, r, parent=None)
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 173, in _add_to_criteria
    if not criterion.candidates:
           ^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\resolvelib\structs.py", line 156, in __bool__
    return bool(self._sequence)
           ^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 155, in __bool__
    return any(self)
           ^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 143, in <genexpr>
    return (c for c in iterator if id(c) not in self._incompatible_ids)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 47, in _iter_built
    candidate = func()
                ^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\factory.py", line 182, in _make_candidate_from_link
    base: Optional[BaseCandidate] = self._make_base_candidate_from_link(
                                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\factory.py", line 228, in _make_base_candidate_from_link
    self._link_candidate_cache[link] = LinkCandidate(
                                       ^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 290, in __init__
    super().__init__(
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 156, in __init__
    self.dist = self._prepare()
                ^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 222, in _prepare
    dist = self._prepare_distribution()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 301, in _prepare_distribution
    return preparer.prepare_linked_requirement(self._ireq, parallel_builds=True)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\operations\prepare.py", line 519, in prepare_linked_requirement
    metadata_dist = self._fetch_metadata_only(req)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\operations\prepare.py", line 371, in _fetch_metadata_only
    return self._fetch_metadata_using_link_data_attr(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\operations\prepare.py", line 391, in _fetch_metadata_using_link_data_attr
    metadata_file = get_http_url(
                    ^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\operations\prepare.py", line 109, in get_http_url
    from_path, content_type = download(link, temp_dir.path)
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\network\download.py", line 134, in __call__
    resp = _http_get_download(self._session, link)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\network\download.py", line 117, in _http_get_download
    resp = session.get(target_url, headers=HEADERS, stream=True)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\sessions.py", line 602, in get
    return self.request("GET", url, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_internal\network\session.py", line 520, in request
    return super().request(method, url, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\sessions.py", line 589, in request
    resp = self.send(prep, **send_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\sessions.py", line 703, in send
    r = adapter.send(request, **kwargs)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\cachecontrol\adapter.py", line 76, in send
    resp = super().send(request, stream, timeout, verify, cert, proxies)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\adapters.py", line 517, in send
    raise SSLError(e, request=request)
pip._vendor.requests.exceptions.SSLError: HTTPSConnectionPool(host='files.pythonhosted.org', port=443): Max retries exceeded with url: /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in
certificate chain (_ssl.c:1000)')))

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

]$ sam build --template template.yaml

Starting Build use cache

Manifest file is changed (new hash: 3298f13049d19cffaa37ca931dd4d421) or dependency folder (.aws-sam\deps\53036498-ac29-43da-b665-6ca82b0cd06d) is missing for (HelloWorldFunction), downloading

dependencies and copying/building source

Building codeuri: C:\labs\aws\Source_Code\lambda\make-thumnail\hello_world runtime: python3.12 metadata: {} architecture: x86_64 functions: HelloWorldFunction

Running PythonPipBuilder:CleanUp

Running PythonPipBuilder:ResolveDependencies

Build Failed

Error: PythonPipBuilder:ResolveDependencies - WARNING: pip is being invoked by an old script wrapper. This will fail in a future version of pip.

Please see https://github.com/pypa/pip/issues/5599 for advice on fixing the underlying issue.

To avoid this problem you can invoke Python with '-m pip' instead of running pip directly.

WARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata

WARNING: Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata

WARNING: Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata

WARNING: Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata

WARNING: Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)'))': /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata

ERROR: Exception:

Traceback (most recent call last):

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 714, in urlopen

httplib_response = self._make_request(

^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 403, in _make_request

self._validate_conn(conn)

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 1053, in _validate_conn

conn.connect()

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connection.py", line 419, in connect

self.sock = ssl_wrap_socket(

^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\util\ssl_.py", line 449, in ssl_wrap_socket

ssl_sock = _ssl_wrap_socket_impl(

^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\util\ssl_.py", line 493, in _ssl_wrap_socket_impl

return ssl_context.wrap_socket(sock, server_hostname=server_hostname)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\Python312\Lib\ssl.py", line 455, in wrap_socket

return self.sslsocket_class._create(

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\Python312\Lib\ssl.py", line 1046, in _create

self.do_handshake()

File "c:\Python312\Lib\ssl.py", line 1321, in do_handshake

self._sslobj.do_handshake()

ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):

File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\adapters.py", line 486, in send

resp = conn.urlopen(

^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 826, in urlopen

return self.urlopen(

^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 826, in urlopen

return self.urlopen(

^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 826, in urlopen

return self.urlopen(

^^^^^^^^^^^^^

[Previous line repeated 2 more times]

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\connectionpool.py", line 798, in urlopen

retries = retries.increment(

^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\urllib3\util\retry.py", line 592, in increment

raise MaxRetryError(_pool, url, error or ResponseError(cause))

pip._vendor.urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='files.pythonhosted.org', port=443): Max retries exceeded with url: /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1000)')))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):

File "c:\labs\aws\Lib\site-packages\pip\_internal\cli\base_command.py", line 180, in exc_logging_wrapper

status = run_func(*args)

^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\cli\req_command.py", line 245, in wrapper

return func(self, options, args)

^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\commands\download.py", line 132, in run

requirement_set = resolver.resolve(reqs, check_supported_wheels=True)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\resolver.py", line 95, in resolve

result = self._result = resolver.resolve(

^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 546, in resolve

state = resolution.resolve(requirements, max_rounds=max_rounds)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 397, in resolve

self._add_to_criteria(self.state.criteria, r, parent=None)

File "c:\labs\aws\Lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 173, in _add_to_criteria

if not criterion.candidates:

^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\resolvelib\structs.py", line 156, in __bool__

return bool(self._sequence)

^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 155, in __bool__

return any(self)

^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 143, in <genexpr>

return (c for c in iterator if id(c) not in self._incompatible_ids)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 47, in _iter_built

candidate = func()

^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\factory.py", line 182, in _make_candidate_from_link

base: Optional[BaseCandidate] = self._make_base_candidate_from_link(

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\factory.py", line 228, in _make_base_candidate_from_link

self._link_candidate_cache[link] = LinkCandidate(

^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 290, in __init__

super().__init__(

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 156, in __init__

self.dist = self._prepare()

^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 222, in _prepare

dist = self._prepare_distribution()

^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 301, in _prepare_distribution

return preparer.prepare_linked_requirement(self._ireq, parallel_builds=True)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\operations\prepare.py", line 519, in prepare_linked_requirement

metadata_dist = self._fetch_metadata_only(req)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\operations\prepare.py", line 371, in _fetch_metadata_only

return self._fetch_metadata_using_link_data_attr(

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\operations\prepare.py", line 391, in _fetch_metadata_using_link_data_attr

metadata_file = get_http_url(

^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\operations\prepare.py", line 109, in get_http_url

from_path, content_type = download(link, temp_dir.path)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\network\download.py", line 134, in __call__

resp = _http_get_download(self._session, link)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\network\download.py", line 117, in _http_get_download

resp = session.get(target_url, headers=HEADERS, stream=True)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\sessions.py", line 602, in get

return self.request("GET", url, **kwargs)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_internal\network\session.py", line 520, in request

return super().request(method, url, *args, **kwargs)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\sessions.py", line 589, in request

resp = self.send(prep, **send_kwargs)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\sessions.py", line 703, in send

r = adapter.send(request, **kwargs)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\cachecontrol\adapter.py", line 76, in send

resp = super().send(request, stream, timeout, verify, cert, proxies)

^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "c:\labs\aws\Lib\site-packages\pip\_vendor\requests\adapters.py", line 517, in send

raise SSLError(e, request=request)

pip._vendor.requests.exceptions.SSLError: HTTPSConnectionPool(host='files.pythonhosted.org', port=443): Max retries exceeded with url: /packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in

certificate chain (_ssl.c:1000)')))

이럴때에는 실행하는 위치에 pip.conf 파일을 다음과 같은 내용으로 만든다.

[global]
trusted-host = pypi.python.org
               pypi.org
               files.pythonhosted.org
               github.com

[global]

trusted-host = pypi.python.org

pypi.org

files.pythonhosted.org

github.com

그리고 이 파일에 대한 환경변수를 지정해 준다.

set PIP_CONFIG_FILE=pip.conf

1	set PIP_CONFIG_FILE=pip.conf

다시 sam build 를 수행하면 정상적으로 실행이 된다.

sam build --template template.yaml

1	sam build --template template.yaml

만일 컨테이너를 사용한다면 다음과 같이 하면 된다.

sam build --use-container --container-env-var PIP_CONFIG_FILE=pip.conf

1	sam build --use-container --container-env-var PIP_CONFIG_FILE=pip.conf

02/20/2024

CodeDeploy, Blue/Green 배포 실패시 인스턴스 삭제.

CodeDeploy, Blue/Green 배포 실패시 Rollback 이 되는데, 이때 인스턴스는 삭제가 되지 않는다. 삭제를 하기위해서는 CloudWatch, SNS 를 통해서 별도의 작업을 해줘야 한다.

출처: If a blue/green deployment fails on AWS, it leaves dangling instances. How do I ensure that old instances are terminated?

02/20/2024

AWS ECS, Service 생성 오류 디버깅

AWS ECS 에서 Cluster 를 생성하고 난 후에 Service 를 생성해야 한다. AWS ECS 는 Cluster, Service 생성시에 CloudFormation 으로 생성하게 되는데, 문제는 오류가 발생했을때다.

“Circuit Breaker” 라고 되어 있어서 그냥 CloudFormation 이 실행하면서 예기치못한 오류가 발생한 것으로 착각할 수 있다. 하지만, 이것이 문제가 아닐 수도 있다.

Service 생성 Event 확인

CloudFormation 에서 “CREATE_FAILED” 라고 되어 있어도 AWS ECS 에 Service 에는 service 가 생성되어 있다. Service 를 클릭하고 들어가서 Events 탭을 클릭하면 생성이 진행되었던 과정을 볼 수 있다.

여기서 task 로 시작하는 해쉬값 링크를 볼 수 있다. 이것을 클릭하고 들어가본다.

이제 뭐가 문제인지를 확실하게 보인다. “pull image mainfest has bean retried 1 time(s)” 로 되어 있고 ECR 에서 이미지를 못가지고 오고 있음을 알 수 있다.

01/08/2024

AWS CloudFront 인증서는 ACM eu-east-1 에서 해야 한다

AWS CloudFront 인증서는 ACM eu-east-1 (N. Virginia) 에서 발급 받아야 한다. 왜냐하면 CloudFront 때문인데, CloudFront 의 메인 리즌이 eu-east-1 로 되어 있어서 다른 리즌에 발급받은 인증서는 CloudFront 에서 사용할 수 없게 된다.

Q: 어느 리전에서 ACM을 사용할 수 있나요?

AWS 서비스를 사용할 수 있는 현재 리전을 확인하려면 AWS 글로벌 인프라 페이지를 참조하세요. Amazon CloudFront에서 ACM 인증서를 사용하려면 미국 동부(버지니아 북부) 리전에서 ACM 인증서를 요청하거나 가져와야 합니다. CloudFront 배포와 연동되어 있는 이 리전의 ACM 인증서는 해당 배포에 대해 구성된 모든 지리적 위치에 배포됩니다.

12/26/2023

Terraform 을 위한 VSCode 세팅

VSCode 에서 Terraform 을 사용하기 위한 세팅은 다음과 같다.

필요한 Extensions

HashiCorp Terraform
IntelliCode
Prettier – Code formatter

위와같은 Extensions 들을 설치하면 Terraform 을 사용하기가 편하다.

사용자 설정

Terraform 을 위한 사용자 설정은 다음과 같다.

    "terraform.languageServer.path": "D:\\Terraform\\terraform.exe",
    "[terraform]": {
        "editor.formatOnSaveMode": "file",
        "editor.formatOnSave": true,
        "editor.defaultFormatter": "hashicorp.terraform",
        "editor.tabSize": 2, // optionally
    },
    "[terraform-vars]": {
        "editor.formatOnSave": true,
        "editor.defaultFormatter": "hashicorp.terraform",
        "editor.formatOnSaveMode": "file",
        "editor.tabSize": 2 // optionally
    }

"terraform.languageServer.path": "D:\\Terraform\\terraform.exe",

"[terraform]": {

"editor.formatOnSaveMode": "file",

"editor.formatOnSave": true,

"editor.defaultFormatter": "hashicorp.terraform",

"editor.tabSize": 2, // optionally

"[terraform-vars]": {

"editor.formatOnSave": true,

"editor.defaultFormatter": "hashicorp.terraform",

"editor.formatOnSaveMode": "file",

"editor.tabSize": 2 // optionally

}

위 설정은 Prettier – Code formatter 가 설치되어 있어야 한다.

12/18/2023

Amazon MSK 2-AZ

Kafka 를 보게 되면서 Amazon MSK 를 보고 있는데, 문서를 보기에는 운영 환경에서 3-AZ 를 권장하고 있다. 성능 문서를 보았을때에도 3-AZ 를 권장하고 있고 Kafka 의 일반적은 환경에서도 3개의 브로커를 사용하도록 하고 있다.

하지만 Amazon MSK 에서는 2-AZ 로도 얼마든지 가능하다고 한다. 2019년 10월 16일 문서에 다음과 같이 되어 있다.

You can now expand your Amazon MSK clusters and deploy new clusters across 2-AZs
The new capability to expand allows your Amazon MSK cluster to scale out as your business grows by dynamically increasing the number of brokers within a cluster. The use of 2-AZ clusters is a good option for customers who already run Apache Kafka brokers in two AZs or use a replication factor of 2 by default.

Replication Factor 를 2로 할 경우에 얼마든지 2-AZ 로 할 수 있다고 되어 있다.

10/08/2022

AWS Athena 를 위한 S3 버킷 정책

AWS ELB 와같은 로그를 AWS S3 에 버킷 저장하는데, 이것을 Athena 에서 읽어서 분석을 하게 된다. 보통 이를 위해서 AWS S3 이 정책을 지정하지 않는 경우가 많은데, 보안상 좋지 않다. 다음과 같이 정책을 지정해주면 된다.

{
    "Sid": "AWSAthenaGetLogs",
    "Effect": "Allow",
    "Principal": {
        "Service": "athena.amazonaws.com"
    },
    "Action": [
        "s3:GetObject",
        "s3:ListBucket",
        "s3:GetBucketLocation"
    ],
    "Resource": [
        "arn:aws:s3:::DOC-EXAMPLE-BUCKET",
        "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"
    ],
    "Condition": {
        "StringEquals": {
            "aws:SourceAccount": "111111111111"
        },
        "ArnLike": {
            "aws:SourceArn": "arn:aws:athena:ap-northeast-2:111111111111:workgroup/athena-vpc-flow-log"
        }
    }
},
{
    "Sid": "AWSAthenaGetLogsFromGlue",
    "Effect": "Allow",
    "Principal": {
        "Service": "glue.amazonaws.com"
    },
    "Action": [
        "s3:GetObject",
        "s3:ListBucket",
        "s3:GetBucketLocation"
    ],
    "Resource": [
        "arn:aws:s3:::DOC-EXAMPLE-BUCKET",
        "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"
    ],
    "Condition": {
        "StringEquals": {
            "aws:SourceAccount": "111111111111"
        }
    }
}

{

"Sid": "AWSAthenaGetLogs",

"Effect": "Allow",

"Principal": {

"Service": "athena.amazonaws.com"

"Action": [

"s3:GetObject",

"s3:ListBucket",

"s3:GetBucketLocation"

"Resource": [

"arn:aws:s3:::DOC-EXAMPLE-BUCKET",

"arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"

"Condition": {

"StringEquals": {

"aws:SourceAccount": "111111111111"

"ArnLike": {

"aws:SourceArn": "arn:aws:athena:ap-northeast-2:111111111111:workgroup/athena-vpc-flow-log"

}

{

"Sid": "AWSAthenaGetLogsFromGlue",

"Effect": "Allow",

"Principal": {

"Service": "glue.amazonaws.com"

"Action": [

"s3:GetObject",

"s3:ListBucket",

"s3:GetBucketLocation"

"Resource": [

"arn:aws:s3:::DOC-EXAMPLE-BUCKET",

"arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"

"Condition": {

"StringEquals": {

"aws:SourceAccount": "111111111111"

}

AWS Athena 는 AWS Glue 를 이용함으로 이와함께 정책을 지정해줘야 한다.

10/08/2022

AWS S3 에서 직접 다운로드 금지하기

AWS S3 의 객체를 직접 다운로드를 보안상 금지해야 하는 경우가 있다. 이를 위해서 AWS S3 에 정책을 다음과 같이 하면 된다.

{
    "Sid": "Deny get requests from Download Files",
    "Effect": "Deny",
    "Principal": "*",
    "Action": "s3:GetObject",
    "Resource": "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*",
    "Condition": {
        "StringLike": {
            "aws:Referer": [
                "https://ap-northeast-2.console.aws.amazon.com/*",
                "https://s3.console.aws.amazon.com/s3/buckets/*",
                "https://s3.console.aws.amazon.com/s3/object/*",
                "https://DOC-EXAMPLE-BUCKET*"
            ]
        }
    }
}

{

"Sid": "Deny get requests from Download Files",

"Effect": "Deny",

"Principal": "*",

"Action": "s3:GetObject",

"Resource": "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*",

"Condition": {

"StringLike": {

"aws:Referer": [

"https://ap-northeast-2.console.aws.amazon.com/*",

"https://s3.console.aws.amazon.com/s3/buckets/*",

"https://s3.console.aws.amazon.com/s3/object/*",

"https://DOC-EXAMPLE-BUCKET*"

]

}

s3:GetObject 액션에 대해서 거부 정책을 적용하고 Referer 를 이용해서 특정 URL 을 지정해주면 된다.

10/08/2022

AWS S3 HTTPS 강제

AWS S3 와 통신을 하는 방법으로 HTTP, HTTPS 두가지 방법이 있다. 하지만 HTTPS 만으로 통신을 하기 위해서는 다음과 같이 정책을 지정해 줘야 한다.

{
    "Sid": "AllowSSLRequestsOnly",
    "Effect": "Deny",
    "Principal": "*",
    "Action": "s3:*",
    "Resource": [
        "arn:aws:s3:::DOC-EXAMPLE-BUCKET",
        "arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"
    ],
    "Condition": {
        "Bool": {
            "aws:SecureTransport": "false"
        }
    }
}

{

"Sid": "AllowSSLRequestsOnly",

"Effect": "Deny",

"Principal": "*",

"Action": "s3:*",

"Resource": [

"arn:aws:s3:::DOC-EXAMPLE-BUCKET",

"arn:aws:s3:::DOC-EXAMPLE-BUCKET/*"

"Condition": {

"Bool": {

"aws:SecureTransport": "false"

}

참고: AWS Config 규칙 s3-bucket-ssl-requests-only를 준수하려면 어떤 S3 버킷 정책을 사용해야 합니까?

08/30/2022

AWS ATHENA 로 VPC FLOW LOG 분석하기 – 2

이전 글에서 AWS 의 Athena 를 이용한 VPC Flow Log 를 어떻게 분석하는지에 대해서 이야기 했다. VPC Flow Log 생성부터, S3 버킷 생성, Athena 데이터베이스와 테이블 그리고 Lambda 를 이용한 파티션 추가까지 비교적 많은 부분을 손봐야 했다.

이 방법은 파티션 작업을 Lambda 를 이용하는 방법으로 하루에 한번 실행시키도록 하고 있다. 하지만, AWS 에서는 이마져도 필요 없는 방법을 제공하는데, 그것이 바로 파티션 프로젝션(Partition Projection) 이다.

AWS 메뉴얼 주의사항

파티션 프로젝션을 하기 위해서 AWS 메뉴얼을 보고 따라했는데 되지 않는다. 정확히는 테이블 생성이 되지만 Athena 에서는 보이지 않는다.

Athena 의 데이터베이스는 Glue 를 이용한다. AWS Glue 가 가보면 Athena 의 테이블을 볼 수 있는데, AWS 메뉴얼대로 파티션 프로젝션을 생성하면 Glue 에는 나오지만 Athena 에는 안나온다. 이는 Glue 에서 S3 저장소를 인식하지 못해 나오는 문제다.

다음의 메뉴얼에는 문제가 있다.

Amazon Kinesis Data Firehose example

일단 위 메뉴얼에는 파티션 프로젝션이 무엇인지를 설명하고 있다.

CREATE EXTERNAL TABLE my_ingested_data (
 ...
)
...
PARTITIONED BY (
 datehour STRING
)
LOCATION "s3://DOC-EXAMPLE-BUCKET/prefix/"
TBLPROPERTIES (
 "projection.enabled" = "true",
 "projection.datehour.type" = "date",
 "projection.datehour.format" = "yyyy/MM/dd/HH",
 "projection.datehour.range" = "2021/01/01/00,NOW",
 "projection.datehour.interval" = "1",
 "projection.datehour.interval.unit" = "HOURS",
 "storage.location.template" = "s3://DOC-EXAMPLE-BUCKET/prefix/${datehour}/"
)

CREATE EXTERNAL TABLE my_ingested_data (

...

)

...

PARTITIONED BY (

datehour STRING

)

LOCATION "s3://DOC-EXAMPLE-BUCKET/prefix/"

TBLPROPERTIES (

"projection.enabled" = "true",

"projection.datehour.type" = "date",

"projection.datehour.format" = "yyyy/MM/dd/HH",

"projection.datehour.range" = "2021/01/01/00,NOW",

"projection.datehour.interval" = "1",

"projection.datehour.interval.unit" = "HOURS",

"storage.location.template" = "s3://DOC-EXAMPLE-BUCKET/prefix/${datehour}/"

)

위 내용을 요약하면 버킷의 구조가 다음과 같다는 것이다.

s3://DOC-EXAMPLE-BUCKET/prefix/2021/01/01/00/
s3://DOC-EXAMPLE-BUCKET/prefix/2021/01/01/01/
s3://DOC-EXAMPLE-BUCKET/prefix/2021/01/01/02/

s3://DOC-EXAMPLE-BUCKET/prefix/2021/01/01/00/

s3://DOC-EXAMPLE-BUCKET/prefix/2021/01/01/01/

s3://DOC-EXAMPLE-BUCKET/prefix/2021/01/01/02/

이런 구조에서 파티션 프로젝션을 걸어주면 자동으로 날짜시간으로 파티션이 형성된다.

VPC Flow Log 의 파티션 프로젝션

그렇다면 이제 VPC Flow Log 에 파티션 프로젝션을 걸어 테이블 생성해 보자.

CREATE EXTERNAL TABLE IF NOT EXISTS `vpc_flow_logs_prj`(
    `version` int,
    `account_id` string,
    `interface_id` string,
    `srcaddr` string,
    `dstaddr` string,
    `srcport` int,
    `dstport` int,
    `protocol` bigint,
    `packets` bigint,
    `bytes` bigint,
    `start` bigint,
    `end` bigint,
    `action` string,
    `log_status` string
)
PARTITIONED BY (`day` string) -- 파티셔닝 컬럼 지정
ROW FORMAT DELIMITED FIELDS TERMINATED BY ' '
LOCATION 's3://flow-log-buckets/AWSLogs/111111222222333333/vpcflowlogs/ap-northeast-2/'
TBLPROPERTIES
(
'projection.enabled' = 'true',
'projection.day.type' = 'date',
'projection.day.range' = '2022/08/23,NOW',
'projection.day.format' = 'yyyy/MM/dd',
'projection.day.interval' = '1',
'projection.day.interval.unit' = 'DAYS'
'storage.location.template' = 's3://flow-log-buckets/AWSLogs/1111112222222333333/vpcflowlogs/ap-northeast-2/${day}/'
)

CREATE EXTERNAL TABLE IF NOT EXISTS `vpc_flow_logs_prj`(

`version` int,

`account_id` string,

`interface_id` string,

`srcaddr` string,

`dstaddr` string,

`srcport` int,

`dstport` int,

`protocol` bigint,

`packets` bigint,

`bytes` bigint,

`start` bigint,

`end` bigint,

`action` string,

`log_status` string

)

PARTITIONED BY (`day` string) -- 파티셔닝 컬럼 지정

ROW FORMAT DELIMITED FIELDS TERMINATED BY ' '

LOCATION 's3://flow-log-buckets/AWSLogs/111111222222333333/vpcflowlogs/ap-northeast-2/'

TBLPROPERTIES

(

'projection.enabled' = 'true',

'projection.day.type' = 'date',

'projection.day.range' = '2022/08/23,NOW',

'projection.day.format' = 'yyyy/MM/dd',

'projection.day.interval' = '1',

'projection.day.interval.unit' = 'DAYS'

'storage.location.template' = 's3://flow-log-buckets/AWSLogs/1111112222222333333/vpcflowlogs/ap-northeast-2/${day}/'

)

위 쿼리문으로 테이블을 생성하면 AWS Glue 에서는 테이블이 생성되지만 Athena 에는 나오지 않는다. 그리고 AWS Clue 에서 테이블 속성을 보면 S3 저장소와 연결되어 있지 않다.

왜 그럴까?

파티션 프로젝션을 연결할때에는 S3 저장소에 저장된 위치는 물론이고 S3 에 저장된 값의 속성도 지정해 줘야 한다. 대표적인 것이 다음과 같은 것이다.

Row format delimited
Stored as inputformat
outputformat

파티션 프로젝션 없이 테이블을 생성할때에는 이 옵션을 지정해 주지 않았다. 그러면 Default 값이 지정되는데, 대부분 잘 맞는 것이였다.

하지만, 파티션 프로젝션을 할 경우에 속성들을 지정해주지 않으면 아무것도 안된다. 다음과 같이 파티션 프로젝션 테이블을 생성해준다.

CREATE EXTERNAL TABLE `vpc_flow_logs_prj`(
  `version` int, 
  `account_id` string, 
  `interface_id` string, 
  `srcaddr` string, 
  `dstaddr` string, 
  `srcport` int, 
  `dstport` int, 
  `protocol` bigint, 
  `packets` bigint, 
  `bytes` bigint, 
  `start` bigint, 
  `end` bigint, 
  `action` string, 
  `log_status` string)
PARTITIONED BY ( 
  `day` string)
ROW FORMAT DELIMITED 
  FIELDS TERMINATED BY ' ' 
STORED AS INPUTFORMAT 
  'org.apache.hadoop.mapred.TextInputFormat' 
OUTPUTFORMAT 
  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
  's3://flow-log-buckets/AWSLogs/111112222223333/vpcflowlogs/ap-northeast-2'
TBLPROPERTIES (
  'skip.header.line.count'='1', 
  'projection.enabled'='true', 
    'projection.day.type' = 'date',
    'projection.day.range' = '2022/08/23,NOW',
    'projection.day.format' = 'yyyy/MM/dd',
    'projection.day.interval' = '1',
    'projection.day.interval.unit' = 'DAYS',
  'storage.location.template'='s3://flow-log-buckets/AWSLogs/111111222222333333/vpcflowlogs/ap-northeast-2/${day}/'
  )

CREATE EXTERNAL TABLE `vpc_flow_logs_prj`(

`version` int,

`account_id` string,

`interface_id` string,

`srcaddr` string,

`dstaddr` string,

`srcport` int,

`dstport` int,

`protocol` bigint,

`packets` bigint,

`bytes` bigint,

`start` bigint,

`end` bigint,

`action` string,

`log_status` string)

PARTITIONED BY (

`day` string)

ROW FORMAT DELIMITED

FIELDS TERMINATED BY ' '

STORED AS INPUTFORMAT

'org.apache.hadoop.mapred.TextInputFormat'

OUTPUTFORMAT

'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'

LOCATION

's3://flow-log-buckets/AWSLogs/111112222223333/vpcflowlogs/ap-northeast-2'

TBLPROPERTIES (

'skip.header.line.count'='1',

'projection.enabled'='true',

'projection.day.type' = 'date',

'projection.day.range' = '2022/08/23,NOW',

'projection.day.format' = 'yyyy/MM/dd',

'projection.day.interval' = '1',

'projection.day.interval.unit' = 'DAYS',

'storage.location.template'='s3://flow-log-buckets/AWSLogs/111111222222333333/vpcflowlogs/ap-northeast-2/${day}/'

)

위와같이 할 경우에 파티션 프로젝션 테이블이 잘 생성된다. 이렇게 생성하고 난 후에 AWS Glue 에서 테이블의 속성을 보게 되면 정상적으로 S3 버킷과 연결이 되어 있고 각종 속성들이 설정되어 있는 것을 볼 수 있다.

2024 4월
일	월	화	수	목	금	토
	1	2	3	4	5	6
7	8	9	10	11	12	13
14	15	16	17	18	19	20
21	22	23	24	25	26	27
28	29	30