99 lines
399 KiB
HTML
99 lines
399 KiB
HTML
|
<!DOCTYPE html>
|
|||
|
|
<html><head><title>数据工程师（data-engineer）</title><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta property="og:title" content="数据工程师（data-engineer）"/><meta property="og:description" content="数据工程师（data-engineer） § 数据工程师（data-engineer）是现代数据驱动的企业中的核心角色，他们专注于设计、构建和维护能够处理各种规模数据集的系统，从 GB 级别到 PB 级别。这些系统通常用于数据存储、查询和分析，支持企业的决策制定、产品创新和运营优化。 职责与工作内容： 数据处理与存储：设计和实施数据存储解决方案，如关系型数据库、Hadoop、HBase 和 Cassandra。 数据流处理与 ETL：使用工具如 Apache Kafka、Apache Flink 进行实时数据流处理，同时设计和实现 ETL 流程。 数据集成：整合来自不同来源的数据，确保数据的一致性和完整性。 性能优化：优化数据查询和处理的性能，确保系统的高效运行。 系统维护与监控：监控数据系统的健康状况，进行故障排除和性能调优。 跨团队协作：与数据科学家、业务分析师、BI 专家和其他团队合作，确保他们可以轻松访问和使用数据。 必备技能与知识： 数据技术栈：熟悉数据库技术（如 MySQL、PostgreSQL）、Hadoop 生态系统（如 HDFS、MapReduce、Hive）和 Spark。 分布式系统：理解分布式系统的原理和挑战，如数据一致性、分区容错等。 编程与脚本：如 Java、Scala、Python 等，用于数据处理和自动化任务。 数据建模与设计：能够设计高效、可扩展的数据模型和架构。 系统监控与维护：使用工具如 Grafana、Prometheus 进行系统监控和告警。 沟通与团队合作：与各团队沟通协作，理解业务需求，为其提供所需的数据支持。 总的来说，数据工程师确保数据系统的稳定、高效和可扩展，从而支持企业做出数据驱动的决策。他们的专业知识和技能，使得企业能够充分利用其数据资产，为客户和业务提供价值。."/><meta property="og:image" content="https://wiki.7wate.com/static/og-image.png"/><meta property="og:width" content="1200"/><meta property="og:height" content="675"/><link rel="icon" href="../../static/icon.png"/><meta name="description" content="数据工程师（data-engineer） § 数据工程师（data-engineer）是现代数据驱动的企业中的核心角色，他们专注于设计、构建和维护能够处理各种规模数据集的系统，从 GB 级别到 PB 级别。这些系统通常用于数据存储、查询和分析，支持企业的决策制定、产品创新和运营优化。 职责与工作内容： 数据处理与存储：设计和实施数据存储解决方案，如关系型数据库、Hadoop、HBase 和 Cassandra。 数据流处理与 ETL：使用工具如 Apache Kafka、Apache Flink 进行实时数据流处理，同时设计和实现 ETL 流程。 数据集成：整合来自不同来源的数据，确保数据的一致性和完整性。 性能优化：优化数据查询和处理的性能，确保系统的高效运行。 系统维护与监控：监控数据系统的健康状况，进行故障排除和性能调优。 跨团队协作：与数据科学家、业务分析师、BI 专家和其他团队合作，确保他们可以轻松访问和使用数据。 必备技能与知识： 数据技术栈：熟悉数据库技术（如 MySQL、PostgreSQL）、Hadoop 生态系统（如 HDFS、MapReduce、Hive）和 Spark。 分布式系统：理解分布式系统的原理和挑战，如数据一致性、分区容错等。 编程与脚本：如 Java、Scala、Python 等，用于数据处理和自动化任务。 数据建模与设计：能够设计高效、可扩展的数据模型和架构。 系统监控与维护：使用工具如 Grafana、Prometheus 进行系统监控和告警。 沟通与团队合作：与各团队沟通协作，理解业务需求，为其提<EFBFBD>
|
|||
|
|
var _hmt = _hmt || [];
|
|||
|
|
(function() {
|
|||
|
|
var hm = document.createElement("script");
|
|||
|
|
hm.src = "https://hm.baidu.com/hm.js?94d8ccb156eb7c65abf317e6e01cdba9";
|
|||
|
|
var s = document.getElementsByTagName("script")[0];
|
|||
|
|
s.parentNode.insertBefore(hm, s);
|
|||
|
|
})();
|
|||
|
|
</script><script async src="https://www.googletagmanager.com/gtag/js?id=G-MHMEL0F832"></script><script>
|
|||
|
|
(function() {
|
|||
|
|
window.dataLayer = window.dataLayer || [];
|
|||
|
|
function gtag() {
|
|||
|
|
window.dataLayer.push(arguments);
|
|||
|
|
}
|
|||
|
|
gtag('js', new Date());
|
|||
|
|
gtag('config', 'G-MHMEL0F832');
|
|||
|
|
})();
|
|||
|
|
</script><link href="../../index.css" rel="stylesheet" type="text/css" spa-preserve/><link href="https://cdn.jsdelivr.net/npm/katex@0.16.0/dist/katex.min.css" rel="stylesheet" type="text/css" spa-preserve/><link href="https://fonts.googleapis.com/css2?family=IBM Plex Mono&family=Schibsted Grotesk:wght@400;700&family=Source Sans Pro:ital,wght@0,400;0,600;1,400;1,600&display=swap" rel="stylesheet" type="text/css" spa-preserve/><script src="../../prescript.js" type="application/javascript" spa-preserve></script><script type="application/javascript" spa-preserve>const fetchData = fetch(`../../static/contentIndex.json`).then(data => data.json())</script></head><body data-slug="Work/data-engineer/home"><div id="quartz-root" class="page"><div id="quartz-body"><div class="left sidebar"><h1 class="page-title "><a href="../..">📚 X·Eden</a></h1><div class="spacer mobile-only"></div><div class="search "><div id="search-icon"><p>Search</p><div></div><svg tabIndex="0" aria-labelledby="title desc" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 19.9 19.7"><title id="title">Search</title><desc id="desc">Search</desc><g class="search-path" fill="none"><path stroke-linecap="square" d="M18.5 18.3l-5.4-5.4"></path><circle cx="8" cy="8" r="7"></circle></g></svg></div><div id="search-container"><div id="search-space"><input autocomplete="off" id="search-bar" name="search" type="text" aria-label="Search for something" placeholder="Search for something"/><div id="results-container"></div></div></div></div><div class="darkmode "><input class="toggle" id="darkmode-toggle" type="checkbox" tabIndex="-1"/><label id="toggle-label-light" for="darkmode-toggle" tabIndex="-1"><svg xmlns="http://www.w3.org/2000/svg" xmlnsXlink="http://www.w3.org/1999/xlink" version="1.1" id="dayIcon" x="0px" y="0px" viewBox="0 0 35 35" style="enable-background:new 0 0 35 35;" xmlSpace="preserve"><title>Light mode</title><path d="M6,17.5C6,16.672,5.328,16,4.5,16h-3C0.672,16,0,16.672,0,17.5 S0.672,19,1.5,19h3C5.328,19,6,18.328,6,17.5z M7.5,26c-0.414,0-0.789,0.168-1.061,0.439l-2,2C4.168,28.711,4,29.086,4,29.5 C4,30.328,4.671,31,5.5,31c0.414,0,0.789-0.168,1.06-0.44l2-2C8.832,28.289,9,27.914,9,27.5C9,26.672,8.329,26,7.5,26z M17.5,6 C18.329,6,19,5.328,19,4.5v-3C19,0.672,18.329,0,17.5,0S16,0.672,16,1.5v3C16,5.328,16.671,6,17.5,6z M27.5,9 c0.414,0,0.789-0.168,1.06-0.439l2-2C30.832,6.289,31,5.914,31,5.5C31,4.672,30.329,4,29.5,4c-0.414,0-0.789,0.168-1.061,0.44 l-2,2C26.168,6.711,26,7.086,26,7.5C26,8.328,26.671,9,27.5,9z M6.439,8.561C6.711,8.832,7.086,9,7.5,9C8.328,9,9,8.328,9,7.5 c0-0.414-0.168-0.789-0.439-1.061l-2-2C6.289,4.168,5.914,4,5.5,4C4.672,4,4,4.672,4,5.5c0,0.414,0.168,0.789,0.439,1.06 L6.439,8.561z M33.5,16h-3c-0.828,0-1.5,0.672-1.5,1.5s0.672,1.5,1.5,1.5h3c0.828,0,1.5-0.672,1.5-1.5S34.328,16,33.5,16z M28.561,26.439C28.289,26.168,27.914,26,27.5,26c-0.828,0-1.5,0.672-1.5,1.5c0,0.414,0.168,0.789,0.439,1.06l2,2 C28.711,30.832,29.086,31,29.5,31c0.828,0,1.5-0.672,1.5-1.5c0-0.414-0.168-0.789-0.439-1.061L28.561,26.439z M17.5,29 c-0.829,0-1.5,0.672-1.5,1.5v3c0,0.828,0.671,1.5,1.5,1.5s1.5-0.672,1.5-1.5v-3C19,29.672,18.329,29,17.5,29z M17.5,7 C11.71,7,7,11.71,7,17.5S11.71,28,17.5,28S28,23.29,28,17.5S23.29,7,17.5,7z M17.5,25c-4.136,0-7.5-3.364-7.5-7.5 c0-4.136,3.364-7.5,7.5-7.5c4.136,0,7.5,3.364,7.5,7.5C25,21.636,21.636,25,17.5,25z"></path></svg></label><label id="toggle-label-dark" for="darkmode-toggle" tabIndex="-1"><svg xmlns="http://www.w3.org/2000/svg" xmlnsXlink="http://www.w3.org/1999/xlink" version="1.1" id="nightIcon" x="0px" y="0px" viewBox="0 0 100 100" style="enable-background='new 0 0 100 100'" xmlSpace="preserve"><title>Dark mode</title><path d="M96.76,66.458c-0.853-0.852-2.15-1.064-3.23-0.534c-6.063,2.991-12.858,4.571-19.655,4.571 C62.022,70.495,50.88,65.88,42.5,57.5C29.043,44.043,25.658,23.536,34.076,6.47c0.532-1.08,0.318-2.379-0.534-3.23 c-0.851-0.852-2.15-1.064-3.23-0.534c-4.918,2.427-9.375,5.619-13.246,9.491c-9.447,9.447-14.65,22.008-14.65,35.369 c0,13.36,5.203,25.921,14.65,35.368s22.008,14
|
|||
|
|
<p>数据工程师（data-engineer）是现代数据驱动的企业中的核心角色，他们专注于设计、构建和维护能够处理各种规模数据集的系统，从 GB 级别到 PB 级别。这些系统通常用于数据存储、查询和分析，支持企业的决策制定、产品创新和运营优化。</p>
|
|||
|
|
<p><strong>职责与工作内容：</strong></p>
|
|||
|
|
<ol>
|
|||
|
|
<li><strong>数据处理与存储</strong>：设计和实施数据存储解决方案，如关系型数据库、Hadoop、HBase 和 Cassandra。</li>
|
|||
|
|
<li><strong>数据流处理与 ETL</strong>：使用工具如 Apache Kafka、Apache Flink 进行实时数据流处理，同时设计和实现 ETL 流程。</li>
|
|||
|
|
<li><strong>数据集成</strong>：整合来自不同来源的数据，确保数据的一致性和完整性。</li>
|
|||
|
|
<li><strong>性能优化</strong>：优化数据查询和处理的性能，确保系统的高效运行。</li>
|
|||
|
|
<li><strong>系统维护与监控</strong>：监控数据系统的健康状况，进行故障排除和性能调优。</li>
|
|||
|
|
<li><strong>跨团队协作</strong>：与数据科学家、业务分析师、BI 专家和其他团队合作，确保他们可以轻松访问和使用数据。</li>
|
|||
|
|
</ol>
|
|||
|
|
<p><strong>必备技能与知识：</strong></p>
|
|||
|
|
<ol>
|
|||
|
|
<li><strong>数据技术栈</strong>：熟悉数据库技术（如 MySQL、PostgreSQL）、Hadoop 生态系统（如 HDFS、MapReduce、Hive）和 Spark。</li>
|
|||
|
|
<li><strong>分布式系统</strong>：理解分布式系统的原理和挑战，如数据一致性、分区容错等。</li>
|
|||
|
|
<li><strong>编程与脚本</strong>：如 Java、Scala、Python 等，用于数据处理和自动化任务。</li>
|
|||
|
|
<li><strong>数据建模与设计</strong>：能够设计高效、可扩展的数据模型和架构。</li>
|
|||
|
|
<li><strong>系统监控与维护</strong>：使用工具如 Grafana、Prometheus 进行系统监控和告警。</li>
|
|||
|
|
<li><strong>沟通与团队合作</strong>：与各团队沟通协作，理解业务需求，为其提供所需的数据支持。</li>
|
|||
|
|
</ol>
|
|||
|
|
<p>总的来说，数据工程师确保数据系统的稳定、高效和可扩展，从而支持企业做出数据驱动的决策。他们的专业知识和技能，使得企业能够充分利用其数据资产，为客户和业务提供价值。</p></article></div><div class="right sidebar"><div class="graph "><h3>Graph View</h3><div class="graph-outer"><div id="graph-container" data-cfg="{"drag":true,"zoom":true,"depth":1,"scale":1.1,"repelForce":0.5,"centerForce":0.3,"linkDistance":30,"fontSize":0.6,"opacityScale":1,"showTags":true,"removeTags":[]}"></div><svg version="1.1" id="global-graph-icon" xmlns="http://www.w3.org/2000/svg" xmlnsXlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 55 55" fill="currentColor" xmlSpace="preserve"><path d="M49,0c-3.309,0-6,2.691-6,6c0,1.035,0.263,2.009,0.726,2.86l-9.829,9.829C32.542,17.634,30.846,17,29,17
|
|||
|
|
s-3.542,0.634-4.898,1.688l-7.669-7.669C16.785,10.424,17,9.74,17,9c0-2.206-1.794-4-4-4S9,6.794,9,9s1.794,4,4,4
|
|||
|
|
c0.74,0,1.424-0.215,2.019-0.567l7.669,7.669C21.634,21.458,21,23.154,21,25s0.634,3.542,1.688,4.897L10.024,42.562
|
|||
|
|
C8.958,41.595,7.549,41,6,41c-3.309,0-6,2.691-6,6s2.691,6,6,6s6-2.691,6-6c0-1.035-0.263-2.009-0.726-2.86l12.829-12.829
|
|||
|
|
c1.106,0.86,2.44,1.436,3.898,1.619v10.16c-2.833,0.478-5,2.942-5,5.91c0,3.309,2.691,6,6,6s6-2.691,6-6c0-2.967-2.167-5.431-5-5.91
|
|||
|
|
v-10.16c1.458-0.183,2.792-0.759,3.898-1.619l7.669,7.669C41.215,39.576,41,40.26,41,41c0,2.206,1.794,4,4,4s4-1.794,4-4
|
|||
|
|
s-1.794-4-4-4c-0.74,0-1.424,0.215-2.019,0.567l-7.669-7.669C36.366,28.542,37,26.846,37,25s-0.634-3.542-1.688-4.897l9.665-9.665
|
|||
|
|
C46.042,11.405,47.451,12,49,12c3.309,0,6-2.691,6-6S52.309,0,49,0z M11,9c0-1.103,0.897-2,2-2s2,0.897,2,2s-0.897,2-2,2
|
|||
|
|
S11,10.103,11,9z M6,51c-2.206,0-4-1.794-4-4s1.794-4,4-4s4,1.794,4,4S8.206,51,6,51z M33,49c0,2.206-1.794,4-4,4s-4-1.794-4-4
|
|||
|
|
s1.794-4,4-4S33,46.794,33,49z M29,31c-3.309,0-6-2.691-6-6s2.691-6,6-6s6,2.691,6,6S32.309,31,29,31z M47,41c0,1.103-0.897,2-2,2
|
|||
|
|
s-2-0.897-2-2s0.897-2,2-2S47,39.897,47,41z M49,10c-2.206,0-4-1.794-4-4s1.794-4,4-4s4,1.794,4,4S51.206,10,49,10z"></path></svg></div><div id="global-graph-outer"><div id="global-graph-container" data-cfg="{"drag":true,"zoom":true,"depth":-1,"scale":0.9,"repelForce":0.5,"centerForce":0.3,"linkDistance":30,"fontSize":0.6,"opacityScale":1,"showTags":true,"removeTags":[]}"></div></div></div><div class="backlinks "><h3>Backlinks</h3><ul class="overflow"><li>No backlinks found</li></ul></div><div class="explorer mobile-only"><button type="button" id="explorer" data-behavior="collapse" data-collapsed="collapsed" data-savestate="true" data-tree="[{"path":"Basics","collapsed":true},{"path":"Basics/english","collapsed":true},{"path":"Blog","collapsed":true},{"path":"Book","collapsed":true},{"path":"Book/个人成长","collapsed":true},{"path":"Book/医学健康","collapsed":true},{"path":"Book/历史","collapsed":true},{"path":"Book/哲学宗教","collapsed":true},{"path":"Book/心理","collapsed":true},{"path":"Book/政治军事","collapsed":true},{"path":"Book/教育学习","collapsed":true},{"path":"Book/文学","collapsed":true},{"path":"Book/生活百科","collapsed":true},{"path":"Book/社会文化","collapsed":true},{"path":"Book/科学技术","collapsed":true},{"path":"Book/经济理财","collapsed":true},{"path":"Book/艺术","collapsed":true},{"path":"Book/计算机","collapsed":true},{"path":"Company","collapsed":true},{"path":"Company/信息化","collapsed":true},{"path":"Company/信息化/规范","collapsed":true},{"path":"Culture","collapsed":true},{"path":"Government","collapsed":true},{"path":"Government/中国","collapsed":true},{"path":"Government/中国/标准","collapsed":true},{"path":"Government/中国/法律","collapsed":true},{"path":"Journal","collapsed":true},{"path":"Journal/2022","collapsed":true},{"path":"Journal/2022/W34","collapsed":true},{"path":"Journal/2022/W35","collapsed":true},{"path":"Journal/2022/W36","collapsed":true},{"path":"Journal/2022/W37","collapsed":true},{"path":"Journal/2022/W38","collapsed":true},{"path":"Journal/2022/W39","collapsed":true},{"path":"Journal/2022/W41","collapsed":true},{"path":"Journal/2022/W42","collapsed":true},{"path":"Journal/2022/W43","collapsed":true},{"path":"Journal/2022/W44","collapsed":true},{"path":"Journal/2022/W45","collapsed":true},{"path":"Journal/2022/W46","collapsed":true},{"path":"Journal/2022/W47","collapsed":true},{"path":"Journal/2022/W48","collapsed":true},{"path":"Journal/2022/W49","collapsed":true},{"path":"Journal/2022/W50","collapsed":true},{"path":"Journal/2023","collapsed":true},{"path":"Jou
|
|||
|
|
function toggleCallout() {
|
|||
|
|
const outerBlock = this.parentElement;
|
|||
|
|
outerBlock.classList.toggle(`is-collapsed`);
|
|||
|
|
const collapsed = outerBlock.classList.contains(`is-collapsed`);
|
|||
|
|
const height = collapsed ? this.scrollHeight : outerBlock.scrollHeight;
|
|||
|
|
outerBlock.style.maxHeight = height + `px`;
|
|||
|
|
let current = outerBlock;
|
|||
|
|
let parent = outerBlock.parentElement;
|
|||
|
|
while (parent) {
|
|||
|
|
if (!parent.classList.contains(`callout`)) {
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
const collapsed2 = parent.classList.contains(`is-collapsed`);
|
|||
|
|
const height2 = collapsed2 ? parent.scrollHeight : parent.scrollHeight + current.scrollHeight;
|
|||
|
|
parent.style.maxHeight = height2 + `px`;
|
|||
|
|
current = parent;
|
|||
|
|
parent = parent.parentElement;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
function setupCallout() {
|
|||
|
|
const collapsible = document.getElementsByClassName(
|
|||
|
|
`callout is-collapsible`
|
|||
|
|
);
|
|||
|
|
for (const div of collapsible) {
|
|||
|
|
const title = div.firstElementChild;
|
|||
|
|
if (title) {
|
|||
|
|
title.removeEventListener(`click`, toggleCallout);
|
|||
|
|
title.addEventListener(`click`, toggleCallout);
|
|||
|
|
const collapsed = div.classList.contains(`is-collapsed`);
|
|||
|
|
const height = collapsed ? title.scrollHeight : div.scrollHeight;
|
|||
|
|
div.style.maxHeight = height + `px`;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
document.addEventListener(`nav`, setupCallout);
|
|||
|
|
window.addEventListener(`resize`, setupCallout);
|
|||
|
|
</script><script type="module">
|
|||
|
|
import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid/dist/mermaid.esm.min.mjs';
|
|||
|
|
const darkMode = document.documentElement.getAttribute('saved-theme') === 'dark'
|
|||
|
|
mermaid.initialize({
|
|||
|
|
startOnLoad: false,
|
|||
|
|
securityLevel: 'loose',
|
|||
|
|
theme: darkMode ? 'dark' : 'default'
|
|||
|
|
});
|
|||
|
|
document.addEventListener('nav', async () => {
|
|||
|
|
await mermaid.run({
|
|||
|
|
querySelector: '.mermaid'
|
|||
|
|
})
|
|||
|
|
});
|
|||
|
|
</script><script src="https://cdn.jsdelivr.net/npm/katex@0.16.7/dist/contrib/copy-tex.min.js" type="application/javascript"></script><script src="../../postscript.js" type="module"></script></html>
|